src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(".tar.gz", "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Bitrig': 'unknown-bitrig',
 181         'Darwin': 'apple-darwin',
 182         'DragonFly': 'unknown-dragonfly',
 183         'FreeBSD': 'unknown-freebsd',
 184         'Haiku': 'unknown-haiku',
 185         'NetBSD': 'unknown-netbsd',
 186         'OpenBSD': 'unknown-openbsd'
 187     }
 188
 189     # Consider the direct transformation first and then the special cases
 190     if ostype in ostype_mapper:
 191         ostype = ostype_mapper[ostype]
 192     elif ostype == 'Linux':
 193         os_from_sp = subprocess.check_output(
 194             ['uname', '-o']).strip().decode(default_encoding)
 195         if os_from_sp == 'Android':
 196             ostype = 'linux-android'
 197         else:
 198             ostype = 'unknown-linux-gnu'
 199     elif ostype == 'SunOS':
 200         ostype = 'sun-solaris'
 201         # On Solaris, uname -m will return a machine classification instead
 202         # of a cpu type, so uname -p is recommended instead.  However, the
 203         # output from that option is too generic for our purposes (it will
 204         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 205         # must be used instead.
 206         try:
 207             cputype = subprocess.check_output(
 208                 ['isainfo', '-k']).strip().decode(default_encoding)
 209         except (subprocess.CalledProcessError, OSError):
 210             err = "isainfo not found"
 211             sys.exit(err)
 212     elif ostype.startswith('MINGW'):
 213         # msys' `uname` does not print gcc configuration, but prints msys
 214         # configuration. so we cannot believe `uname -m`:
 215         # msys1 is always i686 and msys2 is always x86_64.
 216         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 217         # MINGW64 on x86_64.
 218         ostype = 'pc-windows-gnu'
 219         cputype = 'i686'
 220         if os.environ.get('MSYSTEM') == 'MINGW64':
 221             cputype = 'x86_64'
 222     elif ostype.startswith('MSYS'):
 223         ostype = 'pc-windows-gnu'
 224     elif ostype.startswith('CYGWIN_NT'):
 225         cputype = 'i686'
 226         if ostype.endswith('WOW64'):
 227             cputype = 'x86_64'
 228         ostype = 'pc-windows-gnu'
 229     else:
 230         err = "unknown OS type: {}".format(ostype)
 231         sys.exit(err)
 232
 233     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 234         cputype = subprocess.check_output(
 235               ['uname', '-p']).strip().decode(default_encoding)
 236     cputype_mapper = {
 237         'BePC': 'i686',
 238         'aarch64': 'aarch64',
 239         'amd64': 'x86_64',
 240         'arm64': 'aarch64',
 241         'i386': 'i686',
 242         'i486': 'i686',
 243         'i686': 'i686',
 244         'i786': 'i686',
 245         'powerpc': 'powerpc',
 246         'powerpc64': 'powerpc64',
 247         'powerpc64le': 'powerpc64le',
 248         'ppc': 'powerpc',
 249         'ppc64': 'powerpc64',
 250         'ppc64le': 'powerpc64le',
 251         's390x': 's390x',
 252         'x64': 'x86_64',
 253         'x86': 'i686',
 254         'x86-64': 'x86_64',
 255         'x86_64': 'x86_64'
 256     }
 257
 258     # Consider the direct transformation first and then the special cases
 259     if cputype in cputype_mapper:
 260         cputype = cputype_mapper[cputype]
 261     elif cputype in {'xscale', 'arm'}:
 262         cputype = 'arm'
 263         if ostype == 'linux-android':
 264             ostype = 'linux-androideabi'
 265         elif ostype == 'unknown-freebsd':
 266             cputype = subprocess.check_output(
 267                 ['uname', '-p']).strip().decode(default_encoding)
 268             ostype = 'unknown-freebsd'
 269     elif cputype == 'armv6l':
 270         cputype = 'arm'
 271         if ostype == 'linux-android':
 272             ostype = 'linux-androideabi'
 273         else:
 274             ostype += 'eabihf'
 275     elif cputype in {'armv7l', 'armv8l'}:
 276         cputype = 'armv7'
 277         if ostype == 'linux-android':
 278             ostype = 'linux-androideabi'
 279         else:
 280             ostype += 'eabihf'
 281     elif cputype == 'mips':
 282         if sys.byteorder == 'big':
 283             cputype = 'mips'
 284         elif sys.byteorder == 'little':
 285             cputype = 'mipsel'
 286         else:
 287             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 288     elif cputype == 'mips64':
 289         if sys.byteorder == 'big':
 290             cputype = 'mips64'
 291         elif sys.byteorder == 'little':
 292             cputype = 'mips64el'
 293         else:
 294             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 295         # only the n64 ABI is supported, indicate it
 296         ostype += 'abi64'
 297     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 298         pass
 299     else:
 300         err = "unknown cpu type: {}".format(cputype)
 301         sys.exit(err)
 302
 303     return "{}-{}".format(cputype, ostype)
 304
 305
 306 @contextlib.contextmanager
 307 def output(filepath):
 308     tmp = filepath + '.tmp'
 309     with open(tmp, 'w') as f:
 310         yield f
 311     try:
 312         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 313         os.rename(tmp, filepath)
 314     except OSError:
 315         shutil.copy2(tmp, filepath)
 316         os.remove(tmp)
 317
 318
 319 class RustBuild(object):
 320     """Provide all the methods required to build Rust"""
 321     def __init__(self):
 322         self.cargo_channel = ''
 323         self.date = ''
 324         self._download_url = 'https://static.rust-lang.org'
 325         self.rustc_channel = ''
 326         self.build = ''
 327         self.build_dir = os.path.join(os.getcwd(), "build")
 328         self.clean = False
 329         self.config_toml = ''
 330         self.rust_root = ''
 331         self.use_locked_deps = ''
 332         self.use_vendored_sources = ''
 333         self.verbose = False
 334
 335     def download_stage0(self):
 336         """Fetch the build system for Rust, written in Rust
 337
 338         This method will build a cache directory, then it will fetch the
 339         tarball which has the stage0 compiler used to then bootstrap the Rust
 340         compiler itself.
 341
 342         Each downloaded tarball is extracted, after that, the script
 343         will move all the content to the right place.
 344         """
 345         rustc_channel = self.rustc_channel
 346         cargo_channel = self.cargo_channel
 347
 348         if self.rustc().startswith(self.bin_root()) and \
 349                 (not os.path.exists(self.rustc()) or
 350                  self.program_out_of_date(self.rustc_stamp())):
 351             if os.path.exists(self.bin_root()):
 352                 shutil.rmtree(self.bin_root())
 353             filename = "rust-std-{}-{}.tar.gz".format(
 354                 rustc_channel, self.build)
 355             pattern = "rust-std-{}".format(self.build)
 356             self._download_stage0_helper(filename, pattern)
 357
 358             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
 359             self._download_stage0_helper(filename, "rustc")
 360             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 361             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 362             with output(self.rustc_stamp()) as rust_stamp:
 363                 rust_stamp.write(self.date)
 364
 365             # This is required so that we don't mix incompatible MinGW
 366             # libraries/binaries that are included in rust-std with
 367             # the system MinGW ones.
 368             if "pc-windows-gnu" in self.build:
 369                 filename = "rust-mingw-{}-{}.tar.gz".format(
 370                     rustc_channel, self.build)
 371                 self._download_stage0_helper(filename, "rust-mingw")
 372
 373         if self.cargo().startswith(self.bin_root()) and \
 374                 (not os.path.exists(self.cargo()) or
 375                  self.program_out_of_date(self.cargo_stamp())):
 376             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
 377             self._download_stage0_helper(filename, "cargo")
 378             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 379             with output(self.cargo_stamp()) as cargo_stamp:
 380                 cargo_stamp.write(self.date)
 381
 382     def _download_stage0_helper(self, filename, pattern):
 383         cache_dst = os.path.join(self.build_dir, "cache")
 384         rustc_cache = os.path.join(cache_dst, self.date)
 385         if not os.path.exists(rustc_cache):
 386             os.makedirs(rustc_cache)
 387
 388         url = "{}/dist/{}".format(self._download_url, self.date)
 389         tarball = os.path.join(rustc_cache, filename)
 390         if not os.path.exists(tarball):
 391             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 392         unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
 393
 394     @staticmethod
 395     def fix_executable(fname):
 396         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 397
 398         This method is only required on NixOS and uses the PatchELF utility to
 399         change the dynamic linker of ELF executables.
 400
 401         Please see https://nixos.org/patchelf.html for more information
 402         """
 403         default_encoding = sys.getdefaultencoding()
 404         try:
 405             ostype = subprocess.check_output(
 406                 ['uname', '-s']).strip().decode(default_encoding)
 407         except subprocess.CalledProcessError:
 408             return
 409         except OSError as reason:
 410             if getattr(reason, 'winerror', None) is not None:
 411                 return
 412             raise reason
 413
 414         if ostype != "Linux":
 415             return
 416
 417         if not os.path.exists("/etc/NIXOS"):
 418             return
 419         if os.path.exists("/lib"):
 420             return
 421
 422         # At this point we're pretty sure the user is running NixOS
 423         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 424         print(nix_os_msg, fname)
 425
 426         try:
 427             interpreter = subprocess.check_output(
 428                 ["patchelf", "--print-interpreter", fname])
 429             interpreter = interpreter.strip().decode(default_encoding)
 430         except subprocess.CalledProcessError as reason:
 431             print("warning: failed to call patchelf:", reason)
 432             return
 433
 434         loader = interpreter.split("/")[-1]
 435
 436         try:
 437             ldd_output = subprocess.check_output(
 438                 ['ldd', '/run/current-system/sw/bin/sh'])
 439             ldd_output = ldd_output.strip().decode(default_encoding)
 440         except subprocess.CalledProcessError as reason:
 441             print("warning: unable to call ldd:", reason)
 442             return
 443
 444         for line in ldd_output.splitlines():
 445             libname = line.split()[0]
 446             if libname.endswith(loader):
 447                 loader_path = libname[:len(libname) - len(loader)]
 448                 break
 449         else:
 450             print("warning: unable to find the path to the dynamic linker")
 451             return
 452
 453         correct_interpreter = loader_path + loader
 454
 455         try:
 456             subprocess.check_output(
 457                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 458         except subprocess.CalledProcessError as reason:
 459             print("warning: failed to call patchelf:", reason)
 460             return
 461
 462     def rustc_stamp(self):
 463         """Return the path for .rustc-stamp
 464
 465         >>> rb = RustBuild()
 466         >>> rb.build_dir = "build"
 467         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 468         True
 469         """
 470         return os.path.join(self.bin_root(), '.rustc-stamp')
 471
 472     def cargo_stamp(self):
 473         """Return the path for .cargo-stamp
 474
 475         >>> rb = RustBuild()
 476         >>> rb.build_dir = "build"
 477         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 478         True
 479         """
 480         return os.path.join(self.bin_root(), '.cargo-stamp')
 481
 482     def program_out_of_date(self, stamp_path):
 483         """Check if the given program stamp is out of date"""
 484         if not os.path.exists(stamp_path) or self.clean:
 485             return True
 486         with open(stamp_path, 'r') as stamp:
 487             return self.date != stamp.read()
 488
 489     def bin_root(self):
 490         """Return the binary root directory
 491
 492         >>> rb = RustBuild()
 493         >>> rb.build_dir = "build"
 494         >>> rb.bin_root() == os.path.join("build", "stage0")
 495         True
 496
 497         When the 'build' property is given should be a nested directory:
 498
 499         >>> rb.build = "devel"
 500         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 501         True
 502         """
 503         return os.path.join(self.build_dir, self.build, "stage0")
 504
 505     def get_toml(self, key, section=None):
 506         """Returns the value of the given key in config.toml, otherwise returns None
 507
 508         >>> rb = RustBuild()
 509         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 510         >>> rb.get_toml("key2")
 511         'value2'
 512
 513         If the key does not exists, the result is None:
 514
 515         >>> rb.get_toml("key3") is None
 516         True
 517
 518         Optionally also matches the section the key appears in
 519
 520         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 521         >>> rb.get_toml('key', 'a')
 522         'value1'
 523         >>> rb.get_toml('key', 'b')
 524         'value2'
 525         >>> rb.get_toml('key', 'c') is None
 526         True
 527         """
 528
 529         cur_section = None
 530         for line in self.config_toml.splitlines():
 531             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 532             if section_match is not None:
 533                 cur_section = section_match.group(1)
 534
 535             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 536             if match is not None:
 537                 value = match.group(1)
 538                 if section is None or section == cur_section:
 539                     return self.get_string(value) or value.strip()
 540         return None
 541
 542     def cargo(self):
 543         """Return config path for cargo"""
 544         return self.program_config('cargo')
 545
 546     def rustc(self):
 547         """Return config path for rustc"""
 548         return self.program_config('rustc')
 549
 550     def program_config(self, program):
 551         """Return config path for the given program
 552
 553         >>> rb = RustBuild()
 554         >>> rb.config_toml = 'rustc = "rustc"\\n'
 555         >>> rb.program_config('rustc')
 556         'rustc'
 557         >>> rb.config_toml = ''
 558         >>> cargo_path = rb.program_config('cargo')
 559         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 560         ... "bin", "cargo")
 561         True
 562         """
 563         config = self.get_toml(program)
 564         if config:
 565             return os.path.expanduser(config)
 566         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 567             program, self.exe_suffix()))
 568
 569     @staticmethod
 570     def get_string(line):
 571         """Return the value between double quotes
 572
 573         >>> RustBuild.get_string('    "devel"   ')
 574         'devel'
 575         """
 576         start = line.find('"')
 577         if start != -1:
 578             end = start + 1 + line[start + 1:].find('"')
 579             return line[start + 1:end]
 580         start = line.find('\'')
 581         if start != -1:
 582             end = start + 1 + line[start + 1:].find('\'')
 583             return line[start + 1:end]
 584         return None
 585
 586     @staticmethod
 587     def exe_suffix():
 588         """Return a suffix for executables"""
 589         if sys.platform == 'win32':
 590             return '.exe'
 591         return ''
 592
 593     def bootstrap_binary(self):
 594         """Return the path of the bootstrap binary
 595
 596         >>> rb = RustBuild()
 597         >>> rb.build_dir = "build"
 598         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 599         ... "debug", "bootstrap")
 600         True
 601         """
 602         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 603
 604     def build_bootstrap(self):
 605         """Build bootstrap"""
 606         build_dir = os.path.join(self.build_dir, "bootstrap")
 607         if self.clean and os.path.exists(build_dir):
 608             shutil.rmtree(build_dir)
 609         env = os.environ.copy()
 610         env["RUSTC_BOOTSTRAP"] = '1'
 611         env["CARGO_TARGET_DIR"] = build_dir
 612         env["RUSTC"] = self.rustc()
 613         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 614             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 615             if "LD_LIBRARY_PATH" in env else ""
 616         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 617             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 618             if "DYLD_LIBRARY_PATH" in env else ""
 619         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 620             (os.pathsep + env["LIBRARY_PATH"]) \
 621             if "LIBRARY_PATH" in env else ""
 622         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 623
 624         build_section = "target.{}".format(self.build_triple())
 625         target_features = []
 626         if self.get_toml("crt-static", build_section) == "true":
 627             target_features += ["+crt-static"]
 628         elif self.get_toml("crt-static", build_section) == "false":
 629             target_features += ["-crt-static"]
 630         if target_features:
 631             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 632         target_linker = self.get_toml("linker", build_section)
 633         if target_linker is not None:
 634             env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
 635
 636         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 637             os.pathsep + env["PATH"]
 638         if not os.path.isfile(self.cargo()):
 639             raise Exception("no cargo executable found at `{}`".format(
 640                 self.cargo()))
 641         args = [self.cargo(), "build", "--manifest-path",
 642                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 643         for _ in range(1, self.verbose):
 644             args.append("--verbose")
 645         if self.use_locked_deps:
 646             args.append("--locked")
 647         if self.use_vendored_sources:
 648             args.append("--frozen")
 649         run(args, env=env, verbose=self.verbose)
 650
 651     def build_triple(self):
 652         """Build triple as in LLVM"""
 653         config = self.get_toml('build')
 654         if config:
 655             return config
 656         return default_build_triple()
 657
 658     def check_submodule(self, module, slow_submodules):
 659         if not slow_submodules:
 660             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 661                                            cwd=os.path.join(self.rust_root, module),
 662                                            stdout=subprocess.PIPE)
 663             return checked_out
 664         else:
 665             return None
 666
 667     def update_submodule(self, module, checked_out, recorded_submodules):
 668         module_path = os.path.join(self.rust_root, module)
 669
 670         if checked_out != None:
 671             default_encoding = sys.getdefaultencoding()
 672             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 673             if recorded_submodules[module] == checked_out:
 674                 return
 675
 676         print("Updating submodule", module)
 677
 678         run(["git", "submodule", "-q", "sync", module],
 679             cwd=self.rust_root, verbose=self.verbose)
 680         try:
 681             run(["git", "submodule", "update",
 682                  "--init", "--recursive", "--progress", module],
 683                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 684         except RuntimeError:
 685             # Some versions of git don't support --progress.
 686             run(["git", "submodule", "update",
 687                  "--init", "--recursive", module],
 688                 cwd=self.rust_root, verbose=self.verbose)
 689         run(["git", "reset", "-q", "--hard"],
 690             cwd=module_path, verbose=self.verbose)
 691         run(["git", "clean", "-qdfx"],
 692             cwd=module_path, verbose=self.verbose)
 693
 694     def update_submodules(self):
 695         """Update submodules"""
 696         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 697                 self.get_toml('submodules') == "false":
 698             return
 699         slow_submodules = self.get_toml('fast-submodules') == "false"
 700         start_time = time()
 701         if slow_submodules:
 702             print('Unconditionally updating all submodules')
 703         else:
 704             print('Updating only changed submodules')
 705         default_encoding = sys.getdefaultencoding()
 706         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 707             ["git", "config", "--file",
 708              os.path.join(self.rust_root, ".gitmodules"),
 709              "--get-regexp", "path"]
 710         ).decode(default_encoding).splitlines()]
 711         filtered_submodules = []
 712         submodules_names = []
 713         for module in submodules:
 714             if module.endswith("llvm-project"):
 715                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 716                     continue
 717             if module.endswith("llvm-emscripten"):
 718                 backends = self.get_toml('codegen-backends')
 719                 if backends is None or not 'emscripten' in backends:
 720                     continue
 721             check = self.check_submodule(module, slow_submodules)
 722             filtered_submodules.append((module, check))
 723             submodules_names.append(module)
 724         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 725                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 726         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 727         recorded_submodules = {}
 728         for data in recorded:
 729             data = data.split()
 730             recorded_submodules[data[3]] = data[2]
 731         for module in filtered_submodules:
 732             self.update_submodule(module[0], module[1], recorded_submodules)
 733         print("Submodules updated in %.2f seconds" % (time() - start_time))
 734
 735     def set_dev_environment(self):
 736         """Set download URL for development environment"""
 737         self._download_url = 'https://dev-static.rust-lang.org'
 738
 739
 740 def bootstrap(help_triggered):
 741     """Configure, fetch, build and run the initial bootstrap"""
 742
 743     # If the user is asking for help, let them know that the whole download-and-build
 744     # process has to happen before anything is printed out.
 745     if help_triggered:
 746         print("info: Downloading and building bootstrap before processing --help")
 747         print("      command. See src/bootstrap/README.md for help with common")
 748         print("      commands.")
 749
 750     parser = argparse.ArgumentParser(description='Build rust')
 751     parser.add_argument('--config')
 752     parser.add_argument('--build')
 753     parser.add_argument('--src')
 754     parser.add_argument('--clean', action='store_true')
 755     parser.add_argument('-v', '--verbose', action='count', default=0)
 756
 757     args = [a for a in sys.argv if a != '-h' and a != '--help']
 758     args, _ = parser.parse_known_args(args)
 759
 760     # Configure initial bootstrap
 761     build = RustBuild()
 762     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 763     build.verbose = args.verbose
 764     build.clean = args.clean
 765
 766     try:
 767         with open(args.config or 'config.toml') as config:
 768             build.config_toml = config.read()
 769     except (OSError, IOError):
 770         pass
 771
 772     match = re.search(r'\nverbose = (\d+)', build.config_toml)
 773     if match is not None:
 774         build.verbose = max(build.verbose, int(match.group(1)))
 775
 776     build.use_vendored_sources = '\nvendor = true' in build.config_toml
 777
 778     build.use_locked_deps = '\nlocked-deps = true' in build.config_toml
 779
 780     if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
 781         if os.environ.get('USER') != os.environ['SUDO_USER']:
 782             build.use_vendored_sources = True
 783             print('info: looks like you are running this command under `sudo`')
 784             print('      and so in order to preserve your $HOME this will now')
 785             print('      use vendored sources by default. Note that if this')
 786             print('      does not work you should run a normal build first')
 787             print('      before running a command like `sudo ./x.py install`')
 788
 789     if build.use_vendored_sources:
 790         if not os.path.exists('.cargo'):
 791             os.makedirs('.cargo')
 792         with output('.cargo/config') as cargo_config:
 793             cargo_config.write("""
 794                 [source.crates-io]
 795                 replace-with = 'vendored-sources'
 796                 registry = 'https://example.com'
 797
 798                 [source.vendored-sources]
 799                 directory = '{}/vendor'
 800             """.format(build.rust_root))
 801     else:
 802         if os.path.exists('.cargo'):
 803             shutil.rmtree('.cargo')
 804
 805     data = stage0_data(build.rust_root)
 806     build.date = data['date']
 807     build.rustc_channel = data['rustc']
 808     build.cargo_channel = data['cargo']
 809
 810     if 'dev' in data:
 811         build.set_dev_environment()
 812
 813     build.update_submodules()
 814
 815     # Fetch/build the bootstrap
 816     build.build = args.build or build.build_triple()
 817     build.download_stage0()
 818     sys.stdout.flush()
 819     build.build_bootstrap()
 820     sys.stdout.flush()
 821
 822     # Run the bootstrap
 823     args = [build.bootstrap_binary()]
 824     args.extend(sys.argv[1:])
 825     env = os.environ.copy()
 826     env["BUILD"] = build.build
 827     env["SRC"] = build.rust_root
 828     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 829     env["BOOTSTRAP_PYTHON"] = sys.executable
 830     env["BUILD_DIR"] = build.build_dir
 831     env["RUSTC_BOOTSTRAP"] = '1'
 832     env["CARGO"] = build.cargo()
 833     env["RUSTC"] = build.rustc()
 834     run(args, env=env, verbose=build.verbose)
 835
 836
 837 def main():
 838     """Entry point for the bootstrap process"""
 839     start_time = time()
 840
 841     # x.py help <cmd> ...
 842     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 843         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 844
 845     help_triggered = (
 846         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 847     try:
 848         bootstrap(help_triggered)
 849         if not help_triggered:
 850             print("Build completed successfully in {}".format(
 851                 format_build_time(time() - start_time)))
 852     except (SystemExit, KeyboardInterrupt) as error:
 853         if hasattr(error, 'code') and isinstance(error.code, int):
 854             exit_code = error.code
 855         else:
 856             exit_code = 1
 857             print(error)
 858         if not help_triggered:
 859             print("Build completed unsuccessfully in {}".format(
 860                 format_build_time(time() - start_time)))
 861         sys.exit(exit_code)
 862
 863
 864 if __name__ == '__main__':
 865     main()