src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 from __future__ import print_function
  12 import argparse
  13 import contextlib
  14 import datetime
  15 import hashlib
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tarfile
  21 import tempfile
  22
  23 from time import time
  24
  25
  26 def get(url, path, verbose=False):
  27     sha_url = url + ".sha256"
  28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  29         temp_path = temp_file.name
  30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  31         sha_path = sha_file.name
  32
  33     try:
  34         download(sha_path, sha_url, False, verbose)
  35         if os.path.exists(path):
  36             if verify(path, sha_path, False):
  37                 if verbose:
  38                     print("using already-download file " + path)
  39                 return
  40             else:
  41                 if verbose:
  42                     print("ignoring already-download file " + path + " due to failed verification")
  43                 os.unlink(path)
  44         download(temp_path, url, True, verbose)
  45         if not verify(temp_path, sha_path, verbose):
  46             raise RuntimeError("failed verification")
  47         if verbose:
  48             print("moving {} to {}".format(temp_path, path))
  49         shutil.move(temp_path, path)
  50     finally:
  51         delete_if_present(sha_path, verbose)
  52         delete_if_present(temp_path, verbose)
  53
  54
  55 def delete_if_present(path, verbose):
  56     if os.path.isfile(path):
  57         if verbose:
  58             print("removing " + path)
  59         os.unlink(path)
  60
  61
  62 def download(path, url, probably_big, verbose):
  63     for x in range(0, 4):
  64         try:
  65             _download(path, url, probably_big, verbose, True)
  66             return
  67         except RuntimeError:
  68             print("\nspurious failure, trying again")
  69     _download(path, url, probably_big, verbose, False)
  70
  71
  72 def _download(path, url, probably_big, verbose, exception):
  73     if probably_big or verbose:
  74         print("downloading {}".format(url))
  75     # see http://serverfault.com/questions/301128/how-to-download
  76     if sys.platform == 'win32':
  77         run(["PowerShell.exe", "/nologo", "-Command",
  78              "(New-Object System.Net.WebClient)"
  79              ".DownloadFile('{}', '{}')".format(url, path)],
  80             verbose=verbose,
  81             exception=exception)
  82     else:
  83         if probably_big or verbose:
  84             option = "-#"
  85         else:
  86             option = "-s"
  87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
  88             verbose=verbose,
  89             exception=exception)
  90
  91
  92 def verify(path, sha_path, verbose):
  93     if verbose:
  94         print("verifying " + path)
  95     with open(path, "rb") as f:
  96         found = hashlib.sha256(f.read()).hexdigest()
  97     with open(sha_path, "r") as f:
  98         expected = f.readline().split()[0]
  99     verified = found == expected
 100     if not verified:
 101         print("invalid checksum:\n"
 102                "    found:    {}\n"
 103                "    expected: {}".format(found, expected))
 104     return verified
 105
 106
 107 def unpack(tarball, dst, verbose=False, match=None):
 108     print("extracting " + tarball)
 109     fname = os.path.basename(tarball).replace(".tar.gz", "")
 110     with contextlib.closing(tarfile.open(tarball)) as tar:
 111         for p in tar.getnames():
 112             if "/" not in p:
 113                 continue
 114             name = p.replace(fname + "/", "", 1)
 115             if match is not None and not name.startswith(match):
 116                 continue
 117             name = name[len(match) + 1:]
 118
 119             fp = os.path.join(dst, name)
 120             if verbose:
 121                 print("  extracting " + p)
 122             tar.extract(p, dst)
 123             tp = os.path.join(dst, p)
 124             if os.path.isdir(tp) and os.path.exists(fp):
 125                 continue
 126             shutil.move(tp, fp)
 127     shutil.rmtree(os.path.join(dst, fname))
 128
 129 def run(args, verbose=False, exception=False):
 130     if verbose:
 131         print("running: " + ' '.join(args))
 132     sys.stdout.flush()
 133     # Use Popen here instead of call() as it apparently allows powershell on
 134     # Windows to not lock up waiting for input presumably.
 135     ret = subprocess.Popen(args)
 136     code = ret.wait()
 137     if code != 0:
 138         err = "failed to run: " + ' '.join(args)
 139         if verbose or exception:
 140             raise RuntimeError(err)
 141         sys.exit(err)
 142
 143 def stage0_data(rust_root):
 144     nightlies = os.path.join(rust_root, "src/stage0.txt")
 145     data = {}
 146     with open(nightlies, 'r') as nightlies:
 147         for line in nightlies:
 148             line = line.rstrip()  # Strip newline character, '\n'
 149             if line.startswith("#") or line == '':
 150                 continue
 151             a, b = line.split(": ", 1)
 152             data[a] = b
 153     return data
 154
 155 def format_build_time(duration):
 156     return str(datetime.timedelta(seconds=int(duration)))
 157
 158
 159 class RustBuild(object):
 160     def download_stage0(self):
 161         cache_dst = os.path.join(self.build_dir, "cache")
 162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 163         if not os.path.exists(rustc_cache):
 164             os.makedirs(rustc_cache)
 165
 166         channel = self.stage0_rustc_channel()
 167
 168         if self.rustc().startswith(self.bin_root()) and \
 169                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 170             self.print_what_it_means_to_bootstrap()
 171             if os.path.exists(self.bin_root()):
 172                 shutil.rmtree(self.bin_root())
 173             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 174             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 175             tarball = os.path.join(rustc_cache, filename)
 176             if not os.path.exists(tarball):
 177                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 178             unpack(tarball, self.bin_root(),
 179                    match="rust-std-" + self.build,
 180                    verbose=self.verbose)
 181
 182             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 183             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 184             tarball = os.path.join(rustc_cache, filename)
 185             if not os.path.exists(tarball):
 186                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 187             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 188             self.fix_executable(self.bin_root() + "/bin/rustc")
 189             self.fix_executable(self.bin_root() + "/bin/rustdoc")
 190             with open(self.rustc_stamp(), 'w') as f:
 191                 f.write(self.stage0_rustc_date())
 192
 193             if "pc-windows-gnu" in self.build:
 194                 filename = "rust-mingw-{}-{}.tar.gz".format(channel, self.build)
 195                 url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 196                 tarball = os.path.join(rustc_cache, filename)
 197                 if not os.path.exists(tarball):
 198                     get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 199                 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
 200
 201         if self.cargo().startswith(self.bin_root()) and \
 202                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 203             self.print_what_it_means_to_bootstrap()
 204             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
 205             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 206             tarball = os.path.join(rustc_cache, filename)
 207             if not os.path.exists(tarball):
 208                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 209             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 210             self.fix_executable(self.bin_root() + "/bin/cargo")
 211             with open(self.cargo_stamp(), 'w') as f:
 212                 f.write(self.stage0_rustc_date())
 213
 214     def fix_executable(self, fname):
 215         # If we're on NixOS we need to change the path to the dynamic loader
 216
 217         default_encoding = sys.getdefaultencoding()
 218         try:
 219             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 220         except (subprocess.CalledProcessError, WindowsError):
 221             return
 222
 223         if ostype != "Linux":
 224             return
 225
 226         if not os.path.exists("/etc/NIXOS"):
 227             return
 228         if os.path.exists("/lib"):
 229             return
 230
 231         # At this point we're pretty sure the user is running NixOS
 232         print("info: you seem to be running NixOS. Attempting to patch " + fname)
 233
 234         try:
 235             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
 236             interpreter = interpreter.strip().decode(default_encoding)
 237         except subprocess.CalledProcessError as e:
 238             print("warning: failed to call patchelf: %s" % e)
 239             return
 240
 241         loader = interpreter.split("/")[-1]
 242
 243         try:
 244             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
 245             ldd_output = ldd_output.strip().decode(default_encoding)
 246         except subprocess.CalledProcessError as e:
 247             print("warning: unable to call ldd: %s" % e)
 248             return
 249
 250         for line in ldd_output.splitlines():
 251             libname = line.split()[0]
 252             if libname.endswith(loader):
 253                 loader_path = libname[:len(libname) - len(loader)]
 254                 break
 255         else:
 256             print("warning: unable to find the path to the dynamic linker")
 257             return
 258
 259         correct_interpreter = loader_path + loader
 260
 261         try:
 262             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
 263         except subprocess.CalledProcessError as e:
 264             print("warning: failed to call patchelf: %s" % e)
 265             return
 266
 267     def stage0_rustc_date(self):
 268         return self._rustc_date
 269
 270     def stage0_rustc_channel(self):
 271         return self._rustc_channel
 272
 273     def rustc_stamp(self):
 274         return os.path.join(self.bin_root(), '.rustc-stamp')
 275
 276     def cargo_stamp(self):
 277         return os.path.join(self.bin_root(), '.cargo-stamp')
 278
 279     def rustc_out_of_date(self):
 280         if not os.path.exists(self.rustc_stamp()) or self.clean:
 281             return True
 282         with open(self.rustc_stamp(), 'r') as f:
 283             return self.stage0_rustc_date() != f.read()
 284
 285     def cargo_out_of_date(self):
 286         if not os.path.exists(self.cargo_stamp()) or self.clean:
 287             return True
 288         with open(self.cargo_stamp(), 'r') as f:
 289             return self.stage0_rustc_date() != f.read()
 290
 291     def bin_root(self):
 292         return os.path.join(self.build_dir, self.build, "stage0")
 293
 294     def get_toml(self, key):
 295         for line in self.config_toml.splitlines():
 296             if line.startswith(key + ' ='):
 297                 return self.get_string(line)
 298         return None
 299
 300     def get_mk(self, key):
 301         for line in iter(self.config_mk.splitlines()):
 302             if line.startswith(key + ' '):
 303                 var = line[line.find(':=') + 2:].strip()
 304                 if var != '':
 305                     return var
 306         return None
 307
 308     def cargo(self):
 309         config = self.get_toml('cargo')
 310         if config:
 311             return config
 312         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 313         if config:
 314             return config + '/bin/cargo' + self.exe_suffix()
 315         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 316
 317     def rustc(self):
 318         config = self.get_toml('rustc')
 319         if config:
 320             return config
 321         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 322         if config:
 323             return config + '/bin/rustc' + self.exe_suffix()
 324         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 325
 326     def get_string(self, line):
 327         start = line.find('"')
 328         end = start + 1 + line[start + 1:].find('"')
 329         return line[start + 1:end]
 330
 331     def exe_suffix(self):
 332         if sys.platform == 'win32':
 333             return '.exe'
 334         else:
 335             return ''
 336
 337     def print_what_it_means_to_bootstrap(self):
 338         if hasattr(self, 'printed'):
 339             return
 340         self.printed = True
 341         if os.path.exists(self.bootstrap_binary()):
 342             return
 343         if not '--help' in sys.argv or len(sys.argv) == 1:
 344             return
 345
 346         print('info: the build system for Rust is written in Rust, so this')
 347         print('      script is now going to download a stage0 rust compiler')
 348         print('      and then compile the build system itself')
 349         print('')
 350         print('info: in the meantime you can read more about rustbuild at')
 351         print('      src/bootstrap/README.md before the download finishes')
 352
 353     def bootstrap_binary(self):
 354         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
 355
 356     def build_bootstrap(self):
 357         self.print_what_it_means_to_bootstrap()
 358         build_dir = os.path.join(self.build_dir, "bootstrap")
 359         if self.clean and os.path.exists(build_dir):
 360             shutil.rmtree(build_dir)
 361         env = os.environ.copy()
 362         env["CARGO_TARGET_DIR"] = build_dir
 363         env["RUSTC"] = self.rustc()
 364         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 365                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
 366                                  if "LD_LIBRARY_PATH" in env else ""
 367         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 368                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 369                                    if "DYLD_LIBRARY_PATH" in env else ""
 370         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 371                       os.pathsep + env["PATH"]
 372         if not os.path.isfile(self.cargo()):
 373             raise Exception("no cargo executable found at `%s`" % self.cargo())
 374         args = [self.cargo(), "build", "--manifest-path",
 375                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 376         if self.use_locked_deps:
 377             args.append("--locked")
 378         if self.use_vendored_sources:
 379             args.append("--frozen")
 380         self.run(args, env)
 381
 382     def run(self, args, env):
 383         proc = subprocess.Popen(args, env=env)
 384         ret = proc.wait()
 385         if ret != 0:
 386             sys.exit(ret)
 387
 388     def build_triple(self):
 389         default_encoding = sys.getdefaultencoding()
 390         config = self.get_toml('build')
 391         if config:
 392             return config
 393         config = self.get_mk('CFG_BUILD')
 394         if config:
 395             return config
 396         try:
 397             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 398             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 399         except (subprocess.CalledProcessError, OSError):
 400             if sys.platform == 'win32':
 401                 return 'x86_64-pc-windows-msvc'
 402             err = "uname not found"
 403             if self.verbose:
 404                 raise Exception(err)
 405             sys.exit(err)
 406
 407         # The goal here is to come up with the same triple as LLVM would,
 408         # at least for the subset of platforms we're willing to target.
 409         if ostype == 'Linux':
 410             ostype = 'unknown-linux-gnu'
 411         elif ostype == 'FreeBSD':
 412             ostype = 'unknown-freebsd'
 413         elif ostype == 'DragonFly':
 414             ostype = 'unknown-dragonfly'
 415         elif ostype == 'Bitrig':
 416             ostype = 'unknown-bitrig'
 417         elif ostype == 'OpenBSD':
 418             ostype = 'unknown-openbsd'
 419         elif ostype == 'NetBSD':
 420             ostype = 'unknown-netbsd'
 421         elif ostype == 'SunOS':
 422             ostype = 'sun-solaris'
 423             # On Solaris, uname -m will return a machine classification instead
 424             # of a cpu type, so uname -p is recommended instead.  However, the
 425             # output from that option is too generic for our purposes (it will
 426             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 427             # must be used instead.
 428             try:
 429                 cputype = subprocess.check_output(['isainfo',
 430                   '-k']).strip().decode(default_encoding)
 431             except (subprocess.CalledProcessError, OSError):
 432                 err = "isainfo not found"
 433                 if self.verbose:
 434                     raise Exception(err)
 435                 sys.exit(err)
 436         elif ostype == 'Darwin':
 437             ostype = 'apple-darwin'
 438         elif ostype == 'Haiku':
 439             ostype = 'unknown-haiku'
 440         elif ostype.startswith('MINGW'):
 441             # msys' `uname` does not print gcc configuration, but prints msys
 442             # configuration. so we cannot believe `uname -m`:
 443             # msys1 is always i686 and msys2 is always x86_64.
 444             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 445             # MINGW64 on x86_64.
 446             ostype = 'pc-windows-gnu'
 447             cputype = 'i686'
 448             if os.environ.get('MSYSTEM') == 'MINGW64':
 449                 cputype = 'x86_64'
 450         elif ostype.startswith('MSYS'):
 451             ostype = 'pc-windows-gnu'
 452         elif ostype.startswith('CYGWIN_NT'):
 453             cputype = 'i686'
 454             if ostype.endswith('WOW64'):
 455                 cputype = 'x86_64'
 456             ostype = 'pc-windows-gnu'
 457         else:
 458             err = "unknown OS type: " + ostype
 459             if self.verbose:
 460                 raise ValueError(err)
 461             sys.exit(err)
 462
 463         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 464             cputype = 'i686'
 465         elif cputype in {'xscale', 'arm'}:
 466             cputype = 'arm'
 467         elif cputype == 'armv6l':
 468             cputype = 'arm'
 469             ostype += 'eabihf'
 470         elif cputype in {'armv7l', 'armv8l'}:
 471             cputype = 'armv7'
 472             ostype += 'eabihf'
 473         elif cputype == 'aarch64':
 474             cputype = 'aarch64'
 475         elif cputype == 'arm64':
 476             cputype = 'aarch64'
 477         elif cputype == 'mips':
 478             if sys.byteorder == 'big':
 479                 cputype = 'mips'
 480             elif sys.byteorder == 'little':
 481                 cputype = 'mipsel'
 482             else:
 483                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 484         elif cputype == 'mips64':
 485             if sys.byteorder == 'big':
 486                 cputype = 'mips64'
 487             elif sys.byteorder == 'little':
 488                 cputype = 'mips64el'
 489             else:
 490                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 491             # only the n64 ABI is supported, indicate it
 492             ostype += 'abi64'
 493         elif cputype in {'powerpc', 'ppc'}:
 494             cputype = 'powerpc'
 495         elif cputype in {'powerpc64', 'ppc64'}:
 496             cputype = 'powerpc64'
 497         elif cputype in {'powerpc64le', 'ppc64le'}:
 498             cputype = 'powerpc64le'
 499         elif cputype == 'sparcv9':
 500             pass
 501         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 502             cputype = 'x86_64'
 503         elif cputype == 's390x':
 504             cputype = 's390x'
 505         elif cputype == 'BePC':
 506             cputype = 'i686'
 507         else:
 508             err = "unknown cpu type: " + cputype
 509             if self.verbose:
 510                 raise ValueError(err)
 511             sys.exit(err)
 512
 513         return "{}-{}".format(cputype, ostype)
 514
 515 def bootstrap():
 516     parser = argparse.ArgumentParser(description='Build rust')
 517     parser.add_argument('--config')
 518     parser.add_argument('--clean', action='store_true')
 519     parser.add_argument('-v', '--verbose', action='store_true')
 520
 521     args = [a for a in sys.argv if a != '-h' and a != '--help']
 522     args, _ = parser.parse_known_args(args)
 523
 524     # Configure initial bootstrap
 525     rb = RustBuild()
 526     rb.config_toml = ''
 527     rb.config_mk = ''
 528     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 529     rb.build_dir = os.path.join(os.getcwd(), "build")
 530     rb.verbose = args.verbose
 531     rb.clean = args.clean
 532
 533     try:
 534         with open(args.config or 'config.toml') as config:
 535             rb.config_toml = config.read()
 536     except:
 537         pass
 538     try:
 539         rb.config_mk = open('config.mk').read()
 540     except:
 541         pass
 542
 543     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
 544                               'CFG_ENABLE_VENDOR' in rb.config_mk
 545
 546     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
 547                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
 548
 549     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
 550         if os.environ.get('USER') != os.environ['SUDO_USER']:
 551             rb.use_vendored_sources = True
 552             print('info: looks like you are running this command under `sudo`')
 553             print('      and so in order to preserve your $HOME this will now')
 554             print('      use vendored sources by default. Note that if this')
 555             print('      does not work you should run a normal build first')
 556             print('      before running a command like `sudo make install`')
 557
 558     if rb.use_vendored_sources:
 559         if not os.path.exists('.cargo'):
 560             os.makedirs('.cargo')
 561         with open('.cargo/config','w') as f:
 562             f.write("""
 563                 [source.crates-io]
 564                 replace-with = 'vendored-sources'
 565                 registry = 'https://example.com'
 566
 567                 [source.vendored-sources]
 568                 directory = '{}/src/vendor'
 569             """.format(rb.rust_root))
 570     else:
 571         if os.path.exists('.cargo'):
 572             shutil.rmtree('.cargo')
 573
 574     data = stage0_data(rb.rust_root)
 575     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 576
 577     # Fetch/build the bootstrap
 578     rb.build = rb.build_triple()
 579     rb.download_stage0()
 580     sys.stdout.flush()
 581     rb.build_bootstrap()
 582     sys.stdout.flush()
 583
 584     # Run the bootstrap
 585     args = [rb.bootstrap_binary()]
 586     args.extend(sys.argv[1:])
 587     env = os.environ.copy()
 588     env["BUILD"] = rb.build
 589     env["SRC"] = rb.rust_root
 590     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 591     rb.run(args, env)
 592
 593 def main():
 594     start_time = time()
 595     help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 596     try:
 597         bootstrap()
 598         if not help_triggered:
 599             print("Build completed successfully in %s" % format_build_time(time() - start_time))
 600     except (SystemExit, KeyboardInterrupt) as e:
 601         if hasattr(e, 'code') and isinstance(e.code, int):
 602             exit_code = e.code
 603         else:
 604             exit_code = 1
 605             print(e)
 606         if not help_triggered:
 607             print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
 608         sys.exit(exit_code)
 609
 610 if __name__ == '__main__':
 611     main()