src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 from __future__ import print_function
  12 import argparse
  13 import contextlib
  14 import datetime
  15 import hashlib
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tarfile
  21 import tempfile
  22
  23 from time import time
  24
  25
  26 def get(url, path, verbose=False):
  27     sha_url = url + ".sha256"
  28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  29         temp_path = temp_file.name
  30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  31         sha_path = sha_file.name
  32
  33     try:
  34         download(sha_path, sha_url, False, verbose)
  35         if os.path.exists(path):
  36             if verify(path, sha_path, False):
  37                 if verbose:
  38                     print("using already-download file " + path)
  39                 return
  40             else:
  41                 if verbose:
  42                     print("ignoring already-download file " + path + " due to failed verification")
  43                 os.unlink(path)
  44         download(temp_path, url, True, verbose)
  45         if not verify(temp_path, sha_path, verbose):
  46             raise RuntimeError("failed verification")
  47         if verbose:
  48             print("moving {} to {}".format(temp_path, path))
  49         shutil.move(temp_path, path)
  50     finally:
  51         delete_if_present(sha_path, verbose)
  52         delete_if_present(temp_path, verbose)
  53
  54
  55 def delete_if_present(path, verbose):
  56     if os.path.isfile(path):
  57         if verbose:
  58             print("removing " + path)
  59         os.unlink(path)
  60
  61
  62 def download(path, url, probably_big, verbose):
  63     for x in range(0, 4):
  64         try:
  65             _download(path, url, probably_big, verbose, True)
  66             return
  67         except RuntimeError:
  68             print("\nspurious failure, trying again")
  69     _download(path, url, probably_big, verbose, False)
  70
  71
  72 def _download(path, url, probably_big, verbose, exception):
  73     if probably_big or verbose:
  74         print("downloading {}".format(url))
  75     # see http://serverfault.com/questions/301128/how-to-download
  76     if sys.platform == 'win32':
  77         run(["PowerShell.exe", "/nologo", "-Command",
  78              "(New-Object System.Net.WebClient)"
  79              ".DownloadFile('{}', '{}')".format(url, path)],
  80             verbose=verbose,
  81             exception=exception)
  82     else:
  83         if probably_big or verbose:
  84             option = "-#"
  85         else:
  86             option = "-s"
  87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
  88             verbose=verbose,
  89             exception=exception)
  90
  91
  92 def verify(path, sha_path, verbose):
  93     if verbose:
  94         print("verifying " + path)
  95     with open(path, "rb") as f:
  96         found = hashlib.sha256(f.read()).hexdigest()
  97     with open(sha_path, "r") as f:
  98         expected = f.readline().split()[0]
  99     verified = found == expected
 100     if not verified:
 101         print("invalid checksum:\n"
 102                "    found:    {}\n"
 103                "    expected: {}".format(found, expected))
 104     return verified
 105
 106
 107 def unpack(tarball, dst, verbose=False, match=None):
 108     print("extracting " + tarball)
 109     fname = os.path.basename(tarball).replace(".tar.gz", "")
 110     with contextlib.closing(tarfile.open(tarball)) as tar:
 111         for p in tar.getnames():
 112             if "/" not in p:
 113                 continue
 114             name = p.replace(fname + "/", "", 1)
 115             if match is not None and not name.startswith(match):
 116                 continue
 117             name = name[len(match) + 1:]
 118
 119             fp = os.path.join(dst, name)
 120             if verbose:
 121                 print("  extracting " + p)
 122             tar.extract(p, dst)
 123             tp = os.path.join(dst, p)
 124             if os.path.isdir(tp) and os.path.exists(fp):
 125                 continue
 126             shutil.move(tp, fp)
 127     shutil.rmtree(os.path.join(dst, fname))
 128
 129 def run(args, verbose=False, exception=False):
 130     if verbose:
 131         print("running: " + ' '.join(args))
 132     sys.stdout.flush()
 133     # Use Popen here instead of call() as it apparently allows powershell on
 134     # Windows to not lock up waiting for input presumably.
 135     ret = subprocess.Popen(args)
 136     code = ret.wait()
 137     if code != 0:
 138         err = "failed to run: " + ' '.join(args)
 139         if verbose or exception:
 140             raise RuntimeError(err)
 141         sys.exit(err)
 142
 143 def stage0_data(rust_root):
 144     nightlies = os.path.join(rust_root, "src/stage0.txt")
 145     data = {}
 146     with open(nightlies, 'r') as nightlies:
 147         for line in nightlies:
 148             line = line.rstrip()  # Strip newline character, '\n'
 149             if line.startswith("#") or line == '':
 150                 continue
 151             a, b = line.split(": ", 1)
 152             data[a] = b
 153     return data
 154
 155 def format_build_time(duration):
 156     return str(datetime.timedelta(seconds=int(duration)))
 157
 158
 159 class RustBuild(object):
 160     def download_stage0(self):
 161         cache_dst = os.path.join(self.build_dir, "cache")
 162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 163         if not os.path.exists(rustc_cache):
 164             os.makedirs(rustc_cache)
 165
 166         if self.rustc().startswith(self.bin_root()) and \
 167                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 168             self.print_what_it_means_to_bootstrap()
 169             if os.path.exists(self.bin_root()):
 170                 shutil.rmtree(self.bin_root())
 171             channel = self.stage0_rustc_channel()
 172             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 173             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 174             tarball = os.path.join(rustc_cache, filename)
 175             if not os.path.exists(tarball):
 176                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 177             unpack(tarball, self.bin_root(),
 178                    match="rust-std-" + self.build,
 179                    verbose=self.verbose)
 180
 181             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 182             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 183             tarball = os.path.join(rustc_cache, filename)
 184             if not os.path.exists(tarball):
 185                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 186             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 187             self.fix_executable(self.bin_root() + "/bin/rustc")
 188             self.fix_executable(self.bin_root() + "/bin/rustdoc")
 189             with open(self.rustc_stamp(), 'w') as f:
 190                 f.write(self.stage0_rustc_date())
 191
 192         if self.cargo().startswith(self.bin_root()) and \
 193                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 194             self.print_what_it_means_to_bootstrap()
 195             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
 196             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 197             tarball = os.path.join(rustc_cache, filename)
 198             if not os.path.exists(tarball):
 199                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 200             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 201             self.fix_executable(self.bin_root() + "/bin/cargo")
 202             with open(self.cargo_stamp(), 'w') as f:
 203                 f.write(self.stage0_rustc_date())
 204
 205     def fix_executable(self, fname):
 206         # If we're on NixOS we need to change the path to the dynamic loader
 207
 208         default_encoding = sys.getdefaultencoding()
 209         try:
 210             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 211         except (subprocess.CalledProcessError, WindowsError):
 212             return
 213
 214         if ostype != "Linux":
 215             return
 216
 217         if not os.path.exists("/etc/NIXOS"):
 218             return
 219         if os.path.exists("/lib"):
 220             return
 221
 222         # At this point we're pretty sure the user is running NixOS
 223         print("info: you seem to be running NixOS. Attempting to patch " + fname)
 224
 225         try:
 226             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
 227             interpreter = interpreter.strip().decode(default_encoding)
 228         except subprocess.CalledProcessError as e:
 229             print("warning: failed to call patchelf: %s" % e)
 230             return
 231
 232         loader = interpreter.split("/")[-1]
 233
 234         try:
 235             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
 236             ldd_output = ldd_output.strip().decode(default_encoding)
 237         except subprocess.CalledProcessError as e:
 238             print("warning: unable to call ldd: %s" % e)
 239             return
 240
 241         for line in ldd_output.splitlines():
 242             libname = line.split()[0]
 243             if libname.endswith(loader):
 244                 loader_path = libname[:len(libname) - len(loader)]
 245                 break
 246         else:
 247             print("warning: unable to find the path to the dynamic linker")
 248             return
 249
 250         correct_interpreter = loader_path + loader
 251
 252         try:
 253             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
 254         except subprocess.CalledProcessError as e:
 255             print("warning: failed to call patchelf: %s" % e)
 256             return
 257
 258     def stage0_rustc_date(self):
 259         return self._rustc_date
 260
 261     def stage0_rustc_channel(self):
 262         return self._rustc_channel
 263
 264     def rustc_stamp(self):
 265         return os.path.join(self.bin_root(), '.rustc-stamp')
 266
 267     def cargo_stamp(self):
 268         return os.path.join(self.bin_root(), '.cargo-stamp')
 269
 270     def rustc_out_of_date(self):
 271         if not os.path.exists(self.rustc_stamp()) or self.clean:
 272             return True
 273         with open(self.rustc_stamp(), 'r') as f:
 274             return self.stage0_rustc_date() != f.read()
 275
 276     def cargo_out_of_date(self):
 277         if not os.path.exists(self.cargo_stamp()) or self.clean:
 278             return True
 279         with open(self.cargo_stamp(), 'r') as f:
 280             return self.stage0_rustc_date() != f.read()
 281
 282     def bin_root(self):
 283         return os.path.join(self.build_dir, self.build, "stage0")
 284
 285     def get_toml(self, key):
 286         for line in self.config_toml.splitlines():
 287             if line.startswith(key + ' ='):
 288                 return self.get_string(line)
 289         return None
 290
 291     def get_mk(self, key):
 292         for line in iter(self.config_mk.splitlines()):
 293             if line.startswith(key + ' '):
 294                 var = line[line.find(':=') + 2:].strip()
 295                 if var != '':
 296                     return var
 297         return None
 298
 299     def cargo(self):
 300         config = self.get_toml('cargo')
 301         if config:
 302             return config
 303         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 304         if config:
 305             return config + '/bin/cargo' + self.exe_suffix()
 306         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 307
 308     def rustc(self):
 309         config = self.get_toml('rustc')
 310         if config:
 311             return config
 312         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 313         if config:
 314             return config + '/bin/rustc' + self.exe_suffix()
 315         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 316
 317     def get_string(self, line):
 318         start = line.find('"')
 319         end = start + 1 + line[start + 1:].find('"')
 320         return line[start + 1:end]
 321
 322     def exe_suffix(self):
 323         if sys.platform == 'win32':
 324             return '.exe'
 325         else:
 326             return ''
 327
 328     def print_what_it_means_to_bootstrap(self):
 329         if hasattr(self, 'printed'):
 330             return
 331         self.printed = True
 332         if os.path.exists(self.bootstrap_binary()):
 333             return
 334         if not '--help' in sys.argv or len(sys.argv) == 1:
 335             return
 336
 337         print('info: the build system for Rust is written in Rust, so this')
 338         print('      script is now going to download a stage0 rust compiler')
 339         print('      and then compile the build system itself')
 340         print('')
 341         print('info: in the meantime you can read more about rustbuild at')
 342         print('      src/bootstrap/README.md before the download finishes')
 343
 344     def bootstrap_binary(self):
 345         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
 346
 347     def build_bootstrap(self):
 348         self.print_what_it_means_to_bootstrap()
 349         build_dir = os.path.join(self.build_dir, "bootstrap")
 350         if self.clean and os.path.exists(build_dir):
 351             shutil.rmtree(build_dir)
 352         env = os.environ.copy()
 353         env["CARGO_TARGET_DIR"] = build_dir
 354         env["RUSTC"] = self.rustc()
 355         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 356                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
 357                                  if "LD_LIBRARY_PATH" in env else ""
 358         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 359                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 360                                    if "DYLD_LIBRARY_PATH" in env else ""
 361         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 362                       os.pathsep + env["PATH"]
 363         if not os.path.isfile(self.cargo()):
 364             raise Exception("no cargo executable found at `%s`" % self.cargo())
 365         args = [self.cargo(), "build", "--manifest-path",
 366                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 367         if self.use_locked_deps:
 368             args.append("--locked")
 369         if self.use_vendored_sources:
 370             args.append("--frozen")
 371         self.run(args, env)
 372
 373     def run(self, args, env):
 374         proc = subprocess.Popen(args, env=env)
 375         ret = proc.wait()
 376         if ret != 0:
 377             sys.exit(ret)
 378
 379     def build_triple(self):
 380         default_encoding = sys.getdefaultencoding()
 381         config = self.get_toml('build')
 382         if config:
 383             return config
 384         config = self.get_mk('CFG_BUILD')
 385         if config:
 386             return config
 387         try:
 388             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 389             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 390         except (subprocess.CalledProcessError, OSError):
 391             if sys.platform == 'win32':
 392                 return 'x86_64-pc-windows-msvc'
 393             err = "uname not found"
 394             if self.verbose:
 395                 raise Exception(err)
 396             sys.exit(err)
 397
 398         # Darwin's `uname -s` lies and always returns i386. We have to use
 399         # sysctl instead.
 400         if ostype == 'Darwin' and cputype == 'i686':
 401             args = ['sysctl', 'hw.optional.x86_64']
 402             sysctl = subprocess.check_output(args).decode(default_encoding)
 403             if ': 1' in sysctl:
 404                 cputype = 'x86_64'
 405
 406         # The goal here is to come up with the same triple as LLVM would,
 407         # at least for the subset of platforms we're willing to target.
 408         if ostype == 'Linux':
 409             ostype = 'unknown-linux-gnu'
 410         elif ostype == 'FreeBSD':
 411             ostype = 'unknown-freebsd'
 412         elif ostype == 'DragonFly':
 413             ostype = 'unknown-dragonfly'
 414         elif ostype == 'Bitrig':
 415             ostype = 'unknown-bitrig'
 416         elif ostype == 'OpenBSD':
 417             ostype = 'unknown-openbsd'
 418         elif ostype == 'NetBSD':
 419             ostype = 'unknown-netbsd'
 420         elif ostype == 'SunOS':
 421             ostype = 'sun-solaris'
 422             # On Solaris, uname -m will return a machine classification instead
 423             # of a cpu type, so uname -p is recommended instead.  However, the
 424             # output from that option is too generic for our purposes (it will
 425             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 426             # must be used instead.
 427             try:
 428                 cputype = subprocess.check_output(['isainfo',
 429                   '-k']).strip().decode(default_encoding)
 430             except (subprocess.CalledProcessError, OSError):
 431                 err = "isainfo not found"
 432                 if self.verbose:
 433                     raise Exception(err)
 434                 sys.exit(err)
 435         elif ostype == 'Darwin':
 436             ostype = 'apple-darwin'
 437         elif ostype == 'Haiku':
 438             ostype = 'unknown-haiku'
 439         elif ostype.startswith('MINGW'):
 440             # msys' `uname` does not print gcc configuration, but prints msys
 441             # configuration. so we cannot believe `uname -m`:
 442             # msys1 is always i686 and msys2 is always x86_64.
 443             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 444             # MINGW64 on x86_64.
 445             ostype = 'pc-windows-gnu'
 446             cputype = 'i686'
 447             if os.environ.get('MSYSTEM') == 'MINGW64':
 448                 cputype = 'x86_64'
 449         elif ostype.startswith('MSYS'):
 450             ostype = 'pc-windows-gnu'
 451         elif ostype.startswith('CYGWIN_NT'):
 452             cputype = 'i686'
 453             if ostype.endswith('WOW64'):
 454                 cputype = 'x86_64'
 455             ostype = 'pc-windows-gnu'
 456         else:
 457             err = "unknown OS type: " + ostype
 458             if self.verbose:
 459                 raise ValueError(err)
 460             sys.exit(err)
 461
 462         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 463             cputype = 'i686'
 464         elif cputype in {'xscale', 'arm'}:
 465             cputype = 'arm'
 466         elif cputype in {'armv6l', 'armv7l', 'armv8l'}:
 467             cputype = 'arm'
 468             ostype += 'eabihf'
 469         elif cputype == 'armv7l':
 470             cputype = 'armv7'
 471             ostype += 'eabihf'
 472         elif cputype == 'aarch64':
 473             cputype = 'aarch64'
 474         elif cputype == 'arm64':
 475             cputype = 'aarch64'
 476         elif cputype == 'mips':
 477             if sys.byteorder == 'big':
 478                 cputype = 'mips'
 479             elif sys.byteorder == 'little':
 480                 cputype = 'mipsel'
 481             else:
 482                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 483         elif cputype == 'mips64':
 484             if sys.byteorder == 'big':
 485                 cputype = 'mips64'
 486             elif sys.byteorder == 'little':
 487                 cputype = 'mips64el'
 488             else:
 489                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 490             # only the n64 ABI is supported, indicate it
 491             ostype += 'abi64'
 492         elif cputype in {'powerpc', 'ppc'}:
 493             cputype = 'powerpc'
 494         elif cputype in {'powerpc64', 'ppc64'}:
 495             cputype = 'powerpc64'
 496         elif cputype in {'powerpc64le', 'ppc64le'}:
 497             cputype = 'powerpc64le'
 498         elif cputype == 'sparcv9':
 499             pass
 500         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 501             cputype = 'x86_64'
 502         elif cputype == 's390x':
 503             cputype = 's390x'
 504         elif cputype == 'BePC':
 505             cputype = 'i686'
 506         else:
 507             err = "unknown cpu type: " + cputype
 508             if self.verbose:
 509                 raise ValueError(err)
 510             sys.exit(err)
 511
 512         return "{}-{}".format(cputype, ostype)
 513
 514 def bootstrap():
 515     parser = argparse.ArgumentParser(description='Build rust')
 516     parser.add_argument('--config')
 517     parser.add_argument('--clean', action='store_true')
 518     parser.add_argument('-v', '--verbose', action='store_true')
 519
 520     args = [a for a in sys.argv if a != '-h' and a != '--help']
 521     args, _ = parser.parse_known_args(args)
 522
 523     # Configure initial bootstrap
 524     rb = RustBuild()
 525     rb.config_toml = ''
 526     rb.config_mk = ''
 527     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 528     rb.build_dir = os.path.join(os.getcwd(), "build")
 529     rb.verbose = args.verbose
 530     rb.clean = args.clean
 531
 532     try:
 533         with open(args.config or 'config.toml') as config:
 534             rb.config_toml = config.read()
 535     except:
 536         pass
 537     try:
 538         rb.config_mk = open('config.mk').read()
 539     except:
 540         pass
 541
 542     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
 543                               'CFG_ENABLE_VENDOR' in rb.config_mk
 544
 545     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
 546                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
 547
 548     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
 549         if os.environ.get('USER') != os.environ['SUDO_USER']:
 550             rb.use_vendored_sources = True
 551             print('info: looks like you are running this command under `sudo`')
 552             print('      and so in order to preserve your $HOME this will now')
 553             print('      use vendored sources by default. Note that if this')
 554             print('      does not work you should run a normal build first')
 555             print('      before running a command like `sudo make install`')
 556
 557     if rb.use_vendored_sources:
 558         if not os.path.exists('.cargo'):
 559             os.makedirs('.cargo')
 560         with open('.cargo/config','w') as f:
 561             f.write("""
 562                 [source.crates-io]
 563                 replace-with = 'vendored-sources'
 564                 registry = 'https://example.com'
 565
 566                 [source.vendored-sources]
 567                 directory = '{}/src/vendor'
 568             """.format(rb.rust_root))
 569     else:
 570         if os.path.exists('.cargo'):
 571             shutil.rmtree('.cargo')
 572
 573     data = stage0_data(rb.rust_root)
 574     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 575
 576     # Fetch/build the bootstrap
 577     rb.build = rb.build_triple()
 578     rb.download_stage0()
 579     sys.stdout.flush()
 580     rb.build_bootstrap()
 581     sys.stdout.flush()
 582
 583     # Run the bootstrap
 584     args = [rb.bootstrap_binary()]
 585     args.extend(sys.argv[1:])
 586     env = os.environ.copy()
 587     env["BUILD"] = rb.build
 588     env["SRC"] = rb.rust_root
 589     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 590     rb.run(args, env)
 591
 592 def main():
 593     start_time = time()
 594     help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 595     try:
 596         bootstrap()
 597         if not help_triggered:
 598             print("Build completed successfully in %s" % format_build_time(time() - start_time))
 599     except (SystemExit, KeyboardInterrupt) as e:
 600         if hasattr(e, 'code') and isinstance(e.code, int):
 601             exit_code = e.code
 602         else:
 603             exit_code = 1
 604             print(e)
 605         if not help_triggered:
 606             print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
 607         sys.exit(exit_code)
 608
 609 if __name__ == '__main__':
 610     main()