src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 from __future__ import print_function
  12 import argparse
  13 import contextlib
  14 import datetime
  15 import hashlib
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tarfile
  21 import tempfile
  22
  23 from time import time
  24
  25
  26 def get(url, path, verbose=False):
  27     sha_url = url + ".sha256"
  28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  29         temp_path = temp_file.name
  30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  31         sha_path = sha_file.name
  32
  33     try:
  34         download(sha_path, sha_url, False, verbose)
  35         if os.path.exists(path):
  36             if verify(path, sha_path, False):
  37                 if verbose:
  38                     print("using already-download file " + path)
  39                 return
  40             else:
  41                 if verbose:
  42                     print("ignoring already-download file " + path + " due to failed verification")
  43                 os.unlink(path)
  44         download(temp_path, url, True, verbose)
  45         if not verify(temp_path, sha_path, verbose):
  46             raise RuntimeError("failed verification")
  47         if verbose:
  48             print("moving {} to {}".format(temp_path, path))
  49         shutil.move(temp_path, path)
  50     finally:
  51         delete_if_present(sha_path, verbose)
  52         delete_if_present(temp_path, verbose)
  53
  54
  55 def delete_if_present(path, verbose):
  56     if os.path.isfile(path):
  57         if verbose:
  58             print("removing " + path)
  59         os.unlink(path)
  60
  61
  62 def download(path, url, probably_big, verbose):
  63     for x in range(0, 4):
  64         try:
  65             _download(path, url, probably_big, verbose, True)
  66             return
  67         except RuntimeError:
  68             print("\nspurious failure, trying again")
  69     _download(path, url, probably_big, verbose, False)
  70
  71
  72 def _download(path, url, probably_big, verbose, exception):
  73     if probably_big or verbose:
  74         print("downloading {}".format(url))
  75     # see http://serverfault.com/questions/301128/how-to-download
  76     if sys.platform == 'win32':
  77         run(["PowerShell.exe", "/nologo", "-Command",
  78              "(New-Object System.Net.WebClient)"
  79              ".DownloadFile('{}', '{}')".format(url, path)],
  80             verbose=verbose,
  81             exception=exception)
  82     else:
  83         if probably_big or verbose:
  84             option = "-#"
  85         else:
  86             option = "-s"
  87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
  88             verbose=verbose,
  89             exception=exception)
  90
  91
  92 def verify(path, sha_path, verbose):
  93     if verbose:
  94         print("verifying " + path)
  95     with open(path, "rb") as f:
  96         found = hashlib.sha256(f.read()).hexdigest()
  97     with open(sha_path, "r") as f:
  98         expected = f.readline().split()[0]
  99     verified = found == expected
 100     if not verified:
 101         print("invalid checksum:\n"
 102                "    found:    {}\n"
 103                "    expected: {}".format(found, expected))
 104     return verified
 105
 106
 107 def unpack(tarball, dst, verbose=False, match=None):
 108     print("extracting " + tarball)
 109     fname = os.path.basename(tarball).replace(".tar.gz", "")
 110     with contextlib.closing(tarfile.open(tarball)) as tar:
 111         for p in tar.getnames():
 112             if "/" not in p:
 113                 continue
 114             name = p.replace(fname + "/", "", 1)
 115             if match is not None and not name.startswith(match):
 116                 continue
 117             name = name[len(match) + 1:]
 118
 119             fp = os.path.join(dst, name)
 120             if verbose:
 121                 print("  extracting " + p)
 122             tar.extract(p, dst)
 123             tp = os.path.join(dst, p)
 124             if os.path.isdir(tp) and os.path.exists(fp):
 125                 continue
 126             shutil.move(tp, fp)
 127     shutil.rmtree(os.path.join(dst, fname))
 128
 129 def run(args, verbose=False, exception=False):
 130     if verbose:
 131         print("running: " + ' '.join(args))
 132     sys.stdout.flush()
 133     # Use Popen here instead of call() as it apparently allows powershell on
 134     # Windows to not lock up waiting for input presumably.
 135     ret = subprocess.Popen(args)
 136     code = ret.wait()
 137     if code != 0:
 138         err = "failed to run: " + ' '.join(args)
 139         if verbose or exception:
 140             raise RuntimeError(err)
 141         sys.exit(err)
 142
 143 def stage0_data(rust_root):
 144     nightlies = os.path.join(rust_root, "src/stage0.txt")
 145     data = {}
 146     with open(nightlies, 'r') as nightlies:
 147         for line in nightlies:
 148             line = line.rstrip()  # Strip newline character, '\n'
 149             if line.startswith("#") or line == '':
 150                 continue
 151             a, b = line.split(": ", 1)
 152             data[a] = b
 153     return data
 154
 155 def format_build_time(duration):
 156     return str(datetime.timedelta(seconds=int(duration)))
 157
 158
 159 class RustBuild(object):
 160     def download_stage0(self):
 161         cache_dst = os.path.join(self.build_dir, "cache")
 162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 163         cargo_cache = os.path.join(cache_dst, self.stage0_cargo_rev())
 164         if not os.path.exists(rustc_cache):
 165             os.makedirs(rustc_cache)
 166         if not os.path.exists(cargo_cache):
 167             os.makedirs(cargo_cache)
 168
 169         if self.rustc().startswith(self.bin_root()) and \
 170                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 171             self.print_what_it_means_to_bootstrap()
 172             if os.path.exists(self.bin_root()):
 173                 shutil.rmtree(self.bin_root())
 174             channel = self.stage0_rustc_channel()
 175             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 176             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 177             tarball = os.path.join(rustc_cache, filename)
 178             if not os.path.exists(tarball):
 179                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 180             unpack(tarball, self.bin_root(),
 181                    match="rust-std-" + self.build,
 182                    verbose=self.verbose)
 183
 184             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 185             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 186             tarball = os.path.join(rustc_cache, filename)
 187             if not os.path.exists(tarball):
 188                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 189             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 190             self.fix_executable(self.bin_root() + "/bin/rustc")
 191             self.fix_executable(self.bin_root() + "/bin/rustdoc")
 192             with open(self.rustc_stamp(), 'w') as f:
 193                 f.write(self.stage0_rustc_date())
 194
 195         if self.cargo().startswith(self.bin_root()) and \
 196                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 197             self.print_what_it_means_to_bootstrap()
 198             filename = "cargo-nightly-{}.tar.gz".format(self.build)
 199             url = "https://s3.amazonaws.com/rust-lang-ci/cargo-builds/" + self.stage0_cargo_rev()
 200             tarball = os.path.join(cargo_cache, filename)
 201             if not os.path.exists(tarball):
 202                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 203             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 204             self.fix_executable(self.bin_root() + "/bin/cargo")
 205             with open(self.cargo_stamp(), 'w') as f:
 206                 f.write(self.stage0_cargo_rev())
 207
 208     def fix_executable(self, fname):
 209         # If we're on NixOS we need to change the path to the dynamic loader
 210
 211         default_encoding = sys.getdefaultencoding()
 212         try:
 213             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 214         except (subprocess.CalledProcessError, WindowsError):
 215             return
 216
 217         if ostype != "Linux":
 218             return
 219
 220         if not os.path.exists("/etc/NIXOS"):
 221             return
 222         if os.path.exists("/lib"):
 223             return
 224
 225         # At this point we're pretty sure the user is running NixOS
 226         print("info: you seem to be running NixOS. Attempting to patch " + fname)
 227
 228         try:
 229             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
 230             interpreter = interpreter.strip().decode(default_encoding)
 231         except subprocess.CalledProcessError as e:
 232             print("warning: failed to call patchelf: %s" % e)
 233             return
 234
 235         loader = interpreter.split("/")[-1]
 236
 237         try:
 238             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
 239             ldd_output = ldd_output.strip().decode(default_encoding)
 240         except subprocess.CalledProcessError as e:
 241             print("warning: unable to call ldd: %s" % e)
 242             return
 243
 244         for line in ldd_output.splitlines():
 245             libname = line.split()[0]
 246             if libname.endswith(loader):
 247                 loader_path = libname[:len(libname) - len(loader)]
 248                 break
 249         else:
 250             print("warning: unable to find the path to the dynamic linker")
 251             return
 252
 253         correct_interpreter = loader_path + loader
 254
 255         try:
 256             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
 257         except subprocess.CalledProcessError as e:
 258             print("warning: failed to call patchelf: %s" % e)
 259             return
 260
 261     def stage0_cargo_rev(self):
 262         return self._cargo_rev
 263
 264     def stage0_rustc_date(self):
 265         return self._rustc_date
 266
 267     def stage0_rustc_channel(self):
 268         return self._rustc_channel
 269
 270     def rustc_stamp(self):
 271         return os.path.join(self.bin_root(), '.rustc-stamp')
 272
 273     def cargo_stamp(self):
 274         return os.path.join(self.bin_root(), '.cargo-stamp')
 275
 276     def rustc_out_of_date(self):
 277         if not os.path.exists(self.rustc_stamp()) or self.clean:
 278             return True
 279         with open(self.rustc_stamp(), 'r') as f:
 280             return self.stage0_rustc_date() != f.read()
 281
 282     def cargo_out_of_date(self):
 283         if not os.path.exists(self.cargo_stamp()) or self.clean:
 284             return True
 285         with open(self.cargo_stamp(), 'r') as f:
 286             return self.stage0_cargo_rev() != f.read()
 287
 288     def bin_root(self):
 289         return os.path.join(self.build_dir, self.build, "stage0")
 290
 291     def get_toml(self, key):
 292         for line in self.config_toml.splitlines():
 293             if line.startswith(key + ' ='):
 294                 return self.get_string(line)
 295         return None
 296
 297     def get_mk(self, key):
 298         for line in iter(self.config_mk.splitlines()):
 299             if line.startswith(key):
 300                 return line[line.find(':=') + 2:].strip()
 301         return None
 302
 303     def cargo(self):
 304         config = self.get_toml('cargo')
 305         if config:
 306             return config
 307         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 308         if config:
 309             return config + '/bin/cargo' + self.exe_suffix()
 310         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 311
 312     def rustc(self):
 313         config = self.get_toml('rustc')
 314         if config:
 315             return config
 316         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 317         if config:
 318             return config + '/bin/rustc' + self.exe_suffix()
 319         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 320
 321     def get_string(self, line):
 322         start = line.find('"')
 323         end = start + 1 + line[start + 1:].find('"')
 324         return line[start + 1:end]
 325
 326     def exe_suffix(self):
 327         if sys.platform == 'win32':
 328             return '.exe'
 329         else:
 330             return ''
 331
 332     def print_what_it_means_to_bootstrap(self):
 333         if hasattr(self, 'printed'):
 334             return
 335         self.printed = True
 336         if os.path.exists(self.bootstrap_binary()):
 337             return
 338         if not '--help' in sys.argv or len(sys.argv) == 1:
 339             return
 340
 341         print('info: the build system for Rust is written in Rust, so this')
 342         print('      script is now going to download a stage0 rust compiler')
 343         print('      and then compile the build system itself')
 344         print('')
 345         print('info: in the meantime you can read more about rustbuild at')
 346         print('      src/bootstrap/README.md before the download finishes')
 347
 348     def bootstrap_binary(self):
 349         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
 350
 351     def build_bootstrap(self):
 352         self.print_what_it_means_to_bootstrap()
 353         build_dir = os.path.join(self.build_dir, "bootstrap")
 354         if self.clean and os.path.exists(build_dir):
 355             shutil.rmtree(build_dir)
 356         env = os.environ.copy()
 357         env["CARGO_TARGET_DIR"] = build_dir
 358         env["RUSTC"] = self.rustc()
 359         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 360                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
 361                                  if "LD_LIBRARY_PATH" in env else ""
 362         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 363                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 364                                    if "DYLD_LIBRARY_PATH" in env else ""
 365         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 366                       os.pathsep + env["PATH"]
 367         if not os.path.isfile(self.cargo()):
 368             raise Exception("no cargo executable found at `%s`" % self.cargo())
 369         args = [self.cargo(), "build", "--manifest-path",
 370                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 371         if self.use_locked_deps:
 372             args.append("--locked")
 373         if self.use_vendored_sources:
 374             args.append("--frozen")
 375         self.run(args, env)
 376
 377     def run(self, args, env):
 378         proc = subprocess.Popen(args, env=env)
 379         ret = proc.wait()
 380         if ret != 0:
 381             sys.exit(ret)
 382
 383     def build_triple(self):
 384         default_encoding = sys.getdefaultencoding()
 385         config = self.get_toml('build')
 386         if config:
 387             return config
 388         config = self.get_mk('CFG_BUILD')
 389         if config:
 390             return config
 391         try:
 392             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 393             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 394         except (subprocess.CalledProcessError, OSError):
 395             if sys.platform == 'win32':
 396                 return 'x86_64-pc-windows-msvc'
 397             err = "uname not found"
 398             if self.verbose:
 399                 raise Exception(err)
 400             sys.exit(err)
 401
 402         # Darwin's `uname -s` lies and always returns i386. We have to use
 403         # sysctl instead.
 404         if ostype == 'Darwin' and cputype == 'i686':
 405             args = ['sysctl', 'hw.optional.x86_64']
 406             sysctl = subprocess.check_output(args).decode(default_encoding)
 407             if ': 1' in sysctl:
 408                 cputype = 'x86_64'
 409
 410         # The goal here is to come up with the same triple as LLVM would,
 411         # at least for the subset of platforms we're willing to target.
 412         if ostype == 'Linux':
 413             ostype = 'unknown-linux-gnu'
 414         elif ostype == 'FreeBSD':
 415             ostype = 'unknown-freebsd'
 416         elif ostype == 'DragonFly':
 417             ostype = 'unknown-dragonfly'
 418         elif ostype == 'Bitrig':
 419             ostype = 'unknown-bitrig'
 420         elif ostype == 'OpenBSD':
 421             ostype = 'unknown-openbsd'
 422         elif ostype == 'NetBSD':
 423             ostype = 'unknown-netbsd'
 424         elif ostype == 'SunOS':
 425             ostype = 'sun-solaris'
 426             # On Solaris, uname -m will return a machine classification instead
 427             # of a cpu type, so uname -p is recommended instead.  However, the
 428             # output from that option is too generic for our purposes (it will
 429             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 430             # must be used instead.
 431             try:
 432                 cputype = subprocess.check_output(['isainfo',
 433                   '-k']).strip().decode(default_encoding)
 434             except (subprocess.CalledProcessError, OSError):
 435                 err = "isainfo not found"
 436                 if self.verbose:
 437                     raise Exception(err)
 438                 sys.exit(err)
 439         elif ostype == 'Darwin':
 440             ostype = 'apple-darwin'
 441         elif ostype.startswith('MINGW'):
 442             # msys' `uname` does not print gcc configuration, but prints msys
 443             # configuration. so we cannot believe `uname -m`:
 444             # msys1 is always i686 and msys2 is always x86_64.
 445             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 446             # MINGW64 on x86_64.
 447             ostype = 'pc-windows-gnu'
 448             cputype = 'i686'
 449             if os.environ.get('MSYSTEM') == 'MINGW64':
 450                 cputype = 'x86_64'
 451         elif ostype.startswith('MSYS'):
 452             ostype = 'pc-windows-gnu'
 453         elif ostype.startswith('CYGWIN_NT'):
 454             cputype = 'i686'
 455             if ostype.endswith('WOW64'):
 456                 cputype = 'x86_64'
 457             ostype = 'pc-windows-gnu'
 458         else:
 459             err = "unknown OS type: " + ostype
 460             if self.verbose:
 461                 raise ValueError(err)
 462             sys.exit(err)
 463
 464         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 465             cputype = 'i686'
 466         elif cputype in {'xscale', 'arm'}:
 467             cputype = 'arm'
 468         elif cputype in {'armv7l', 'armv8l'}:
 469             cputype = 'arm'
 470             ostype += 'eabihf'
 471         elif cputype == 'aarch64':
 472             cputype = 'aarch64'
 473         elif cputype == 'arm64':
 474             cputype = 'aarch64'
 475         elif cputype == 'mips':
 476             if sys.byteorder == 'big':
 477                 cputype = 'mips'
 478             elif sys.byteorder == 'little':
 479                 cputype = 'mipsel'
 480             else:
 481                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 482         elif cputype == 'mips64':
 483             if sys.byteorder == 'big':
 484                 cputype = 'mips64'
 485             elif sys.byteorder == 'little':
 486                 cputype = 'mips64el'
 487             else:
 488                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 489             # only the n64 ABI is supported, indicate it
 490             ostype += 'abi64'
 491         elif cputype in {'powerpc', 'ppc', 'ppc64'}:
 492             cputype = 'powerpc'
 493         elif cputype == 'sparcv9':
 494             pass
 495         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 496             cputype = 'x86_64'
 497         else:
 498             err = "unknown cpu type: " + cputype
 499             if self.verbose:
 500                 raise ValueError(err)
 501             sys.exit(err)
 502
 503         return "{}-{}".format(cputype, ostype)
 504
 505 def bootstrap():
 506     parser = argparse.ArgumentParser(description='Build rust')
 507     parser.add_argument('--config')
 508     parser.add_argument('--clean', action='store_true')
 509     parser.add_argument('-v', '--verbose', action='store_true')
 510
 511     args = [a for a in sys.argv if a != '-h' and a != '--help']
 512     args, _ = parser.parse_known_args(args)
 513
 514     # Configure initial bootstrap
 515     rb = RustBuild()
 516     rb.config_toml = ''
 517     rb.config_mk = ''
 518     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 519     rb.build_dir = os.path.join(os.getcwd(), "build")
 520     rb.verbose = args.verbose
 521     rb.clean = args.clean
 522
 523     try:
 524         with open(args.config or 'config.toml') as config:
 525             rb.config_toml = config.read()
 526     except:
 527         pass
 528     try:
 529         rb.config_mk = open('config.mk').read()
 530     except:
 531         pass
 532
 533     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
 534                               'CFG_ENABLE_VENDOR' in rb.config_mk
 535
 536     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
 537                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
 538
 539     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
 540         if os.environ.get('USER') != os.environ['SUDO_USER']:
 541             rb.use_vendored_sources = True
 542             print('info: looks like you are running this command under `sudo`')
 543             print('      and so in order to preserve your $HOME this will now')
 544             print('      use vendored sources by default. Note that if this')
 545             print('      does not work you should run a normal build first')
 546             print('      before running a command like `sudo make install`')
 547
 548     if rb.use_vendored_sources:
 549         if not os.path.exists('.cargo'):
 550             os.makedirs('.cargo')
 551         with open('.cargo/config','w') as f:
 552             f.write("""
 553                 [source.crates-io]
 554                 replace-with = 'vendored-sources'
 555                 registry = 'https://example.com'
 556
 557                 [source.vendored-sources]
 558                 directory = '{}/src/vendor'
 559             """.format(rb.rust_root))
 560     else:
 561         if os.path.exists('.cargo'):
 562             shutil.rmtree('.cargo')
 563
 564     data = stage0_data(rb.rust_root)
 565     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 566     rb._cargo_rev = data['cargo']
 567
 568     # Fetch/build the bootstrap
 569     rb.build = rb.build_triple()
 570     rb.download_stage0()
 571     sys.stdout.flush()
 572     rb.build_bootstrap()
 573     sys.stdout.flush()
 574
 575     # Run the bootstrap
 576     args = [rb.bootstrap_binary()]
 577     args.extend(sys.argv[1:])
 578     env = os.environ.copy()
 579     env["BUILD"] = rb.build
 580     env["SRC"] = rb.rust_root
 581     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 582     rb.run(args, env)
 583
 584 def main():
 585     start_time = time()
 586     try:
 587         bootstrap()
 588         print("Build completed successfully in %s" % format_build_time(time() - start_time))
 589     except (SystemExit, KeyboardInterrupt) as e:
 590         if hasattr(e, 'code') and isinstance(e.code, int):
 591             exit_code = e.code
 592         else:
 593             exit_code = 1
 594             print(e)
 595         print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
 596         sys.exit(exit_code)
 597
 598 if __name__ == '__main__':
 599     main()