src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(".tar.gz", "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = 'https://static.rust-lang.org'
 324         self.rustc_channel = ''
 325         self.build = ''
 326         self.build_dir = os.path.join(os.getcwd(), "build")
 327         self.clean = False
 328         self.config_toml = ''
 329         self.rust_root = ''
 330         self.use_locked_deps = ''
 331         self.use_vendored_sources = ''
 332         self.verbose = False
 333
 334     def download_stage0(self):
 335         """Fetch the build system for Rust, written in Rust
 336
 337         This method will build a cache directory, then it will fetch the
 338         tarball which has the stage0 compiler used to then bootstrap the Rust
 339         compiler itself.
 340
 341         Each downloaded tarball is extracted, after that, the script
 342         will move all the content to the right place.
 343         """
 344         rustc_channel = self.rustc_channel
 345         cargo_channel = self.cargo_channel
 346
 347         if self.rustc().startswith(self.bin_root()) and \
 348                 (not os.path.exists(self.rustc()) or
 349                  self.program_out_of_date(self.rustc_stamp())):
 350             if os.path.exists(self.bin_root()):
 351                 shutil.rmtree(self.bin_root())
 352             filename = "rust-std-{}-{}.tar.gz".format(
 353                 rustc_channel, self.build)
 354             pattern = "rust-std-{}".format(self.build)
 355             self._download_stage0_helper(filename, pattern)
 356
 357             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
 358             self._download_stage0_helper(filename, "rustc")
 359             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 360             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 361             with output(self.rustc_stamp()) as rust_stamp:
 362                 rust_stamp.write(self.date)
 363
 364             # This is required so that we don't mix incompatible MinGW
 365             # libraries/binaries that are included in rust-std with
 366             # the system MinGW ones.
 367             if "pc-windows-gnu" in self.build:
 368                 filename = "rust-mingw-{}-{}.tar.gz".format(
 369                     rustc_channel, self.build)
 370                 self._download_stage0_helper(filename, "rust-mingw")
 371
 372         if self.cargo().startswith(self.bin_root()) and \
 373                 (not os.path.exists(self.cargo()) or
 374                  self.program_out_of_date(self.cargo_stamp())):
 375             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
 376             self._download_stage0_helper(filename, "cargo")
 377             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 378             with output(self.cargo_stamp()) as cargo_stamp:
 379                 cargo_stamp.write(self.date)
 380
 381     def _download_stage0_helper(self, filename, pattern):
 382         cache_dst = os.path.join(self.build_dir, "cache")
 383         rustc_cache = os.path.join(cache_dst, self.date)
 384         if not os.path.exists(rustc_cache):
 385             os.makedirs(rustc_cache)
 386
 387         url = "{}/dist/{}".format(self._download_url, self.date)
 388         tarball = os.path.join(rustc_cache, filename)
 389         if not os.path.exists(tarball):
 390             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 391         unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
 392
 393     @staticmethod
 394     def fix_executable(fname):
 395         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 396
 397         This method is only required on NixOS and uses the PatchELF utility to
 398         change the dynamic linker of ELF executables.
 399
 400         Please see https://nixos.org/patchelf.html for more information
 401         """
 402         default_encoding = sys.getdefaultencoding()
 403         try:
 404             ostype = subprocess.check_output(
 405                 ['uname', '-s']).strip().decode(default_encoding)
 406         except subprocess.CalledProcessError:
 407             return
 408         except OSError as reason:
 409             if getattr(reason, 'winerror', None) is not None:
 410                 return
 411             raise reason
 412
 413         if ostype != "Linux":
 414             return
 415
 416         if not os.path.exists("/etc/NIXOS"):
 417             return
 418         if os.path.exists("/lib"):
 419             return
 420
 421         # At this point we're pretty sure the user is running NixOS
 422         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 423         print(nix_os_msg, fname)
 424
 425         try:
 426             interpreter = subprocess.check_output(
 427                 ["patchelf", "--print-interpreter", fname])
 428             interpreter = interpreter.strip().decode(default_encoding)
 429         except subprocess.CalledProcessError as reason:
 430             print("warning: failed to call patchelf:", reason)
 431             return
 432
 433         loader = interpreter.split("/")[-1]
 434
 435         try:
 436             ldd_output = subprocess.check_output(
 437                 ['ldd', '/run/current-system/sw/bin/sh'])
 438             ldd_output = ldd_output.strip().decode(default_encoding)
 439         except subprocess.CalledProcessError as reason:
 440             print("warning: unable to call ldd:", reason)
 441             return
 442
 443         for line in ldd_output.splitlines():
 444             libname = line.split()[0]
 445             if libname.endswith(loader):
 446                 loader_path = libname[:len(libname) - len(loader)]
 447                 break
 448         else:
 449             print("warning: unable to find the path to the dynamic linker")
 450             return
 451
 452         correct_interpreter = loader_path + loader
 453
 454         try:
 455             subprocess.check_output(
 456                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 457         except subprocess.CalledProcessError as reason:
 458             print("warning: failed to call patchelf:", reason)
 459             return
 460
 461     def rustc_stamp(self):
 462         """Return the path for .rustc-stamp
 463
 464         >>> rb = RustBuild()
 465         >>> rb.build_dir = "build"
 466         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 467         True
 468         """
 469         return os.path.join(self.bin_root(), '.rustc-stamp')
 470
 471     def cargo_stamp(self):
 472         """Return the path for .cargo-stamp
 473
 474         >>> rb = RustBuild()
 475         >>> rb.build_dir = "build"
 476         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 477         True
 478         """
 479         return os.path.join(self.bin_root(), '.cargo-stamp')
 480
 481     def program_out_of_date(self, stamp_path):
 482         """Check if the given program stamp is out of date"""
 483         if not os.path.exists(stamp_path) or self.clean:
 484             return True
 485         with open(stamp_path, 'r') as stamp:
 486             return self.date != stamp.read()
 487
 488     def bin_root(self):
 489         """Return the binary root directory
 490
 491         >>> rb = RustBuild()
 492         >>> rb.build_dir = "build"
 493         >>> rb.bin_root() == os.path.join("build", "stage0")
 494         True
 495
 496         When the 'build' property is given should be a nested directory:
 497
 498         >>> rb.build = "devel"
 499         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 500         True
 501         """
 502         return os.path.join(self.build_dir, self.build, "stage0")
 503
 504     def get_toml(self, key, section=None):
 505         """Returns the value of the given key in config.toml, otherwise returns None
 506
 507         >>> rb = RustBuild()
 508         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 509         >>> rb.get_toml("key2")
 510         'value2'
 511
 512         If the key does not exists, the result is None:
 513
 514         >>> rb.get_toml("key3") is None
 515         True
 516
 517         Optionally also matches the section the key appears in
 518
 519         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 520         >>> rb.get_toml('key', 'a')
 521         'value1'
 522         >>> rb.get_toml('key', 'b')
 523         'value2'
 524         >>> rb.get_toml('key', 'c') is None
 525         True
 526         """
 527
 528         cur_section = None
 529         for line in self.config_toml.splitlines():
 530             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 531             if section_match is not None:
 532                 cur_section = section_match.group(1)
 533
 534             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 535             if match is not None:
 536                 value = match.group(1)
 537                 if section is None or section == cur_section:
 538                     return self.get_string(value) or value.strip()
 539         return None
 540
 541     def cargo(self):
 542         """Return config path for cargo"""
 543         return self.program_config('cargo')
 544
 545     def rustc(self):
 546         """Return config path for rustc"""
 547         return self.program_config('rustc')
 548
 549     def program_config(self, program):
 550         """Return config path for the given program
 551
 552         >>> rb = RustBuild()
 553         >>> rb.config_toml = 'rustc = "rustc"\\n'
 554         >>> rb.program_config('rustc')
 555         'rustc'
 556         >>> rb.config_toml = ''
 557         >>> cargo_path = rb.program_config('cargo')
 558         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 559         ... "bin", "cargo")
 560         True
 561         """
 562         config = self.get_toml(program)
 563         if config:
 564             return os.path.expanduser(config)
 565         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 566             program, self.exe_suffix()))
 567
 568     @staticmethod
 569     def get_string(line):
 570         """Return the value between double quotes
 571
 572         >>> RustBuild.get_string('    "devel"   ')
 573         'devel'
 574         """
 575         start = line.find('"')
 576         if start != -1:
 577             end = start + 1 + line[start + 1:].find('"')
 578             return line[start + 1:end]
 579         start = line.find('\'')
 580         if start != -1:
 581             end = start + 1 + line[start + 1:].find('\'')
 582             return line[start + 1:end]
 583         return None
 584
 585     @staticmethod
 586     def exe_suffix():
 587         """Return a suffix for executables"""
 588         if sys.platform == 'win32':
 589             return '.exe'
 590         return ''
 591
 592     def bootstrap_binary(self):
 593         """Return the path of the bootstrap binary
 594
 595         >>> rb = RustBuild()
 596         >>> rb.build_dir = "build"
 597         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 598         ... "debug", "bootstrap")
 599         True
 600         """
 601         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 602
 603     def build_bootstrap(self):
 604         """Build bootstrap"""
 605         build_dir = os.path.join(self.build_dir, "bootstrap")
 606         if self.clean and os.path.exists(build_dir):
 607             shutil.rmtree(build_dir)
 608         env = os.environ.copy()
 609         env["RUSTC_BOOTSTRAP"] = '1'
 610         env["CARGO_TARGET_DIR"] = build_dir
 611         env["RUSTC"] = self.rustc()
 612         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 613             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 614             if "LD_LIBRARY_PATH" in env else ""
 615         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 616             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 617             if "DYLD_LIBRARY_PATH" in env else ""
 618         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 619             (os.pathsep + env["LIBRARY_PATH"]) \
 620             if "LIBRARY_PATH" in env else ""
 621         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 622
 623         build_section = "target.{}".format(self.build_triple())
 624         target_features = []
 625         if self.get_toml("crt-static", build_section) == "true":
 626             target_features += ["+crt-static"]
 627         elif self.get_toml("crt-static", build_section) == "false":
 628             target_features += ["-crt-static"]
 629         if target_features:
 630             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 631         target_linker = self.get_toml("linker", build_section)
 632         if target_linker is not None:
 633             env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
 634
 635         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 636             os.pathsep + env["PATH"]
 637         if not os.path.isfile(self.cargo()):
 638             raise Exception("no cargo executable found at `{}`".format(
 639                 self.cargo()))
 640         args = [self.cargo(), "build", "--manifest-path",
 641                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 642         for _ in range(1, self.verbose):
 643             args.append("--verbose")
 644         if self.use_locked_deps:
 645             args.append("--locked")
 646         if self.use_vendored_sources:
 647             args.append("--frozen")
 648         run(args, env=env, verbose=self.verbose)
 649
 650     def build_triple(self):
 651         """Build triple as in LLVM"""
 652         config = self.get_toml('build')
 653         if config:
 654             return config
 655         return default_build_triple()
 656
 657     def check_submodule(self, module, slow_submodules):
 658         if not slow_submodules:
 659             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 660                                            cwd=os.path.join(self.rust_root, module),
 661                                            stdout=subprocess.PIPE)
 662             return checked_out
 663         else:
 664             return None
 665
 666     def update_submodule(self, module, checked_out, recorded_submodules):
 667         module_path = os.path.join(self.rust_root, module)
 668
 669         if checked_out != None:
 670             default_encoding = sys.getdefaultencoding()
 671             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 672             if recorded_submodules[module] == checked_out:
 673                 return
 674
 675         print("Updating submodule", module)
 676
 677         run(["git", "submodule", "-q", "sync", module],
 678             cwd=self.rust_root, verbose=self.verbose)
 679         try:
 680             run(["git", "submodule", "update",
 681                  "--init", "--recursive", "--progress", module],
 682                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 683         except RuntimeError:
 684             # Some versions of git don't support --progress.
 685             run(["git", "submodule", "update",
 686                  "--init", "--recursive", module],
 687                 cwd=self.rust_root, verbose=self.verbose)
 688         run(["git", "reset", "-q", "--hard"],
 689             cwd=module_path, verbose=self.verbose)
 690         run(["git", "clean", "-qdfx"],
 691             cwd=module_path, verbose=self.verbose)
 692
 693     def update_submodules(self):
 694         """Update submodules"""
 695         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 696                 self.get_toml('submodules') == "false":
 697             return
 698         slow_submodules = self.get_toml('fast-submodules') == "false"
 699         start_time = time()
 700         if slow_submodules:
 701             print('Unconditionally updating all submodules')
 702         else:
 703             print('Updating only changed submodules')
 704         default_encoding = sys.getdefaultencoding()
 705         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 706             ["git", "config", "--file",
 707              os.path.join(self.rust_root, ".gitmodules"),
 708              "--get-regexp", "path"]
 709         ).decode(default_encoding).splitlines()]
 710         filtered_submodules = []
 711         submodules_names = []
 712         for module in submodules:
 713             if module.endswith("llvm-project"):
 714                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 715                     continue
 716             if module.endswith("llvm-emscripten"):
 717                 backends = self.get_toml('codegen-backends')
 718                 if backends is None or not 'emscripten' in backends:
 719                     continue
 720             check = self.check_submodule(module, slow_submodules)
 721             filtered_submodules.append((module, check))
 722             submodules_names.append(module)
 723         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 724                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 725         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 726         recorded_submodules = {}
 727         for data in recorded:
 728             data = data.split()
 729             recorded_submodules[data[3]] = data[2]
 730         for module in filtered_submodules:
 731             self.update_submodule(module[0], module[1], recorded_submodules)
 732         print("Submodules updated in %.2f seconds" % (time() - start_time))
 733
 734     def set_dev_environment(self):
 735         """Set download URL for development environment"""
 736         self._download_url = 'https://dev-static.rust-lang.org'
 737
 738
 739 def bootstrap(help_triggered):
 740     """Configure, fetch, build and run the initial bootstrap"""
 741
 742     # If the user is asking for help, let them know that the whole download-and-build
 743     # process has to happen before anything is printed out.
 744     if help_triggered:
 745         print("info: Downloading and building bootstrap before processing --help")
 746         print("      command. See src/bootstrap/README.md for help with common")
 747         print("      commands.")
 748
 749     parser = argparse.ArgumentParser(description='Build rust')
 750     parser.add_argument('--config')
 751     parser.add_argument('--build')
 752     parser.add_argument('--src')
 753     parser.add_argument('--clean', action='store_true')
 754     parser.add_argument('-v', '--verbose', action='count', default=0)
 755
 756     args = [a for a in sys.argv if a != '-h' and a != '--help']
 757     args, _ = parser.parse_known_args(args)
 758
 759     # Configure initial bootstrap
 760     build = RustBuild()
 761     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 762     build.verbose = args.verbose
 763     build.clean = args.clean
 764
 765     try:
 766         with open(args.config or 'config.toml') as config:
 767             build.config_toml = config.read()
 768     except (OSError, IOError):
 769         pass
 770
 771     match = re.search(r'\nverbose = (\d+)', build.config_toml)
 772     if match is not None:
 773         build.verbose = max(build.verbose, int(match.group(1)))
 774
 775     build.use_vendored_sources = '\nvendor = true' in build.config_toml
 776
 777     build.use_locked_deps = '\nlocked-deps = true' in build.config_toml
 778
 779     if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
 780         if os.environ.get('USER') != os.environ['SUDO_USER']:
 781             build.use_vendored_sources = True
 782             print('info: looks like you are running this command under `sudo`')
 783             print('      and so in order to preserve your $HOME this will now')
 784             print('      use vendored sources by default. Note that if this')
 785             print('      does not work you should run a normal build first')
 786             print('      before running a command like `sudo ./x.py install`')
 787
 788     if build.use_vendored_sources:
 789         if not os.path.exists('.cargo'):
 790             os.makedirs('.cargo')
 791         with output('.cargo/config') as cargo_config:
 792             cargo_config.write("""
 793                 [source.crates-io]
 794                 replace-with = 'vendored-sources'
 795                 registry = 'https://example.com'
 796
 797                 [source.vendored-sources]
 798                 directory = '{}/vendor'
 799             """.format(build.rust_root))
 800     else:
 801         if os.path.exists('.cargo'):
 802             shutil.rmtree('.cargo')
 803
 804     data = stage0_data(build.rust_root)
 805     build.date = data['date']
 806     build.rustc_channel = data['rustc']
 807     build.cargo_channel = data['cargo']
 808
 809     if 'dev' in data:
 810         build.set_dev_environment()
 811
 812     build.update_submodules()
 813
 814     # Fetch/build the bootstrap
 815     build.build = args.build or build.build_triple()
 816     build.download_stage0()
 817     sys.stdout.flush()
 818     build.build_bootstrap()
 819     sys.stdout.flush()
 820
 821     # Run the bootstrap
 822     args = [build.bootstrap_binary()]
 823     args.extend(sys.argv[1:])
 824     env = os.environ.copy()
 825     env["BUILD"] = build.build
 826     env["SRC"] = build.rust_root
 827     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 828     env["BOOTSTRAP_PYTHON"] = sys.executable
 829     env["BUILD_DIR"] = build.build_dir
 830     env["RUSTC_BOOTSTRAP"] = '1'
 831     env["CARGO"] = build.cargo()
 832     env["RUSTC"] = build.rustc()
 833     run(args, env=env, verbose=build.verbose)
 834
 835
 836 def main():
 837     """Entry point for the bootstrap process"""
 838     start_time = time()
 839
 840     # x.py help <cmd> ...
 841     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 842         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 843
 844     help_triggered = (
 845         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 846     try:
 847         bootstrap(help_triggered)
 848         if not help_triggered:
 849             print("Build completed successfully in {}".format(
 850                 format_build_time(time() - start_time)))
 851     except (SystemExit, KeyboardInterrupt) as error:
 852         if hasattr(error, 'code') and isinstance(error.code, int):
 853             exit_code = error.code
 854         else:
 855             exit_code = 1
 856             print(error)
 857         if not help_triggered:
 858             print("Build completed unsuccessfully in {}".format(
 859                 format_build_time(time() - start_time)))
 860         sys.exit(exit_code)
 861
 862
 863 if __name__ == '__main__':
 864     main()