src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time
  17
  18 def support_xz():
  19     try:
  20         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21             temp_path = temp_file.name
  22         with tarfile.open(temp_path, "w:xz"):
  23             pass
  24         return True
  25     except tarfile.CompressionError:
  26         return False
  27
  28 def get(base, url, path, checksums, verbose=False, do_verify=True):
  29     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  30         temp_path = temp_file.name
  31
  32     try:
  33         if do_verify:
  34             if url not in checksums:
  35                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  36             sha256 = checksums[url]
  37             if os.path.exists(path):
  38                 if verify(path, sha256, False):
  39                     if verbose:
  40                         print("using already-download file", path)
  41                     return
  42                 else:
  43                     if verbose:
  44                         print("ignoring already-download file",
  45                             path, "due to failed verification")
  46                     os.unlink(path)
  47         download(temp_path, "{}/{}".format(base, url), True, verbose)
  48         if do_verify and not verify(temp_path, sha256, verbose):
  49             raise RuntimeError("failed verification")
  50         if verbose:
  51             print("moving {} to {}".format(temp_path, path))
  52         shutil.move(temp_path, path)
  53     finally:
  54         if os.path.isfile(temp_path):
  55             if verbose:
  56                 print("removing", temp_path)
  57             os.unlink(temp_path)
  58
  59
  60 def download(path, url, probably_big, verbose):
  61     for _ in range(0, 4):
  62         try:
  63             _download(path, url, probably_big, verbose, True)
  64             return
  65         except RuntimeError:
  66             print("\nspurious failure, trying again")
  67     _download(path, url, probably_big, verbose, False)
  68
  69
  70 def _download(path, url, probably_big, verbose, exception):
  71     if probably_big or verbose:
  72         print("downloading {}".format(url))
  73     # see https://serverfault.com/questions/301128/how-to-download
  74     if sys.platform == 'win32':
  75         run(["PowerShell.exe", "/nologo", "-Command",
  76              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  77              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  78             verbose=verbose,
  79             exception=exception)
  80     else:
  81         if probably_big or verbose:
  82             option = "-#"
  83         else:
  84             option = "-s"
  85         require(["curl", "--version"])
  86         run(["curl", option,
  87              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  88              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  89              "--retry", "3", "-Sf", "-o", path, url],
  90             verbose=verbose,
  91             exception=exception)
  92
  93
  94 def verify(path, expected, verbose):
  95     """Check if the sha256 sum of the given path is valid"""
  96     if verbose:
  97         print("verifying", path)
  98     with open(path, "rb") as source:
  99         found = hashlib.sha256(source.read()).hexdigest()
 100     verified = found == expected
 101     if not verified:
 102         print("invalid checksum:\n"
 103               "    found:    {}\n"
 104               "    expected: {}".format(found, expected))
 105     return verified
 106
 107
 108 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 109     """Unpack the given tarball file"""
 110     print("extracting", tarball)
 111     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 112     with contextlib.closing(tarfile.open(tarball)) as tar:
 113         for member in tar.getnames():
 114             if "/" not in member:
 115                 continue
 116             name = member.replace(fname + "/", "", 1)
 117             if match is not None and not name.startswith(match):
 118                 continue
 119             name = name[len(match) + 1:]
 120
 121             dst_path = os.path.join(dst, name)
 122             if verbose:
 123                 print("  extracting", member)
 124             tar.extract(member, dst)
 125             src_path = os.path.join(dst, member)
 126             if os.path.isdir(src_path) and os.path.exists(dst_path):
 127                 continue
 128             shutil.move(src_path, dst_path)
 129     shutil.rmtree(os.path.join(dst, fname))
 130
 131
 132 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
 133     """Run a child program in a new process"""
 134     if verbose:
 135         print("running: " + ' '.join(args))
 136     sys.stdout.flush()
 137     # Use Popen here instead of call() as it apparently allows powershell on
 138     # Windows to not lock up waiting for input presumably.
 139     ret = subprocess.Popen(args, **kwargs)
 140     code = ret.wait()
 141     if code != 0:
 142         err = "failed to run: " + ' '.join(args)
 143         if verbose or exception:
 144             raise RuntimeError(err)
 145         # For most failures, we definitely do want to print this error, or the user will have no
 146         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 147         # have already printed an error above, so there's no need to print the exact command we're
 148         # running.
 149         if is_bootstrap:
 150             sys.exit(1)
 151         else:
 152             sys.exit(err)
 153
 154
 155 def require(cmd, exit=True):
 156     '''Run a command, returning its output.
 157     On error,
 158         If `exit` is `True`, exit the process.
 159         Otherwise, return None.'''
 160     try:
 161         return subprocess.check_output(cmd).strip()
 162     except (subprocess.CalledProcessError, OSError) as exc:
 163         if not exit:
 164             return None
 165         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 166         print("Please make sure it's installed and in the path.")
 167         sys.exit(1)
 168
 169
 170 def format_build_time(duration):
 171     """Return a nicer format for build time
 172
 173     >>> format_build_time('300')
 174     '0:05:00'
 175     """
 176     return str(datetime.timedelta(seconds=int(duration)))
 177
 178
 179 def default_build_triple(verbose):
 180     """Build triple as in LLVM"""
 181     # If the user already has a host build triple with an existing `rustc`
 182     # install, use their preference. This fixes most issues with Windows builds
 183     # being detected as GNU instead of MSVC.
 184     default_encoding = sys.getdefaultencoding()
 185     try:
 186         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 187                 stderr=subprocess.DEVNULL)
 188         version = version.decode(default_encoding)
 189         host = next(x for x in version.split('\n') if x.startswith("host: "))
 190         triple = host.split("host: ")[1]
 191         if verbose:
 192             print("detected default triple {} from pre-installed rustc".format(triple))
 193         return triple
 194     except Exception as e:
 195         if verbose:
 196             print("pre-installed rustc not detected: {}".format(e))
 197             print("falling back to auto-detect")
 198
 199     required = sys.platform != 'win32'
 200     ostype = require(["uname", "-s"], exit=required)
 201     cputype = require(['uname', '-m'], exit=required)
 202
 203     # If we do not have `uname`, assume Windows.
 204     if ostype is None or cputype is None:
 205         return 'x86_64-pc-windows-msvc'
 206
 207     ostype = ostype.decode(default_encoding)
 208     cputype = cputype.decode(default_encoding)
 209
 210     # The goal here is to come up with the same triple as LLVM would,
 211     # at least for the subset of platforms we're willing to target.
 212     ostype_mapper = {
 213         'Darwin': 'apple-darwin',
 214         'DragonFly': 'unknown-dragonfly',
 215         'FreeBSD': 'unknown-freebsd',
 216         'Haiku': 'unknown-haiku',
 217         'NetBSD': 'unknown-netbsd',
 218         'OpenBSD': 'unknown-openbsd'
 219     }
 220
 221     # Consider the direct transformation first and then the special cases
 222     if ostype in ostype_mapper:
 223         ostype = ostype_mapper[ostype]
 224     elif ostype == 'Linux':
 225         os_from_sp = subprocess.check_output(
 226             ['uname', '-o']).strip().decode(default_encoding)
 227         if os_from_sp == 'Android':
 228             ostype = 'linux-android'
 229         else:
 230             ostype = 'unknown-linux-gnu'
 231     elif ostype == 'SunOS':
 232         ostype = 'pc-solaris'
 233         # On Solaris, uname -m will return a machine classification instead
 234         # of a cpu type, so uname -p is recommended instead.  However, the
 235         # output from that option is too generic for our purposes (it will
 236         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 237         # must be used instead.
 238         cputype = require(['isainfo', '-k']).decode(default_encoding)
 239         # sparc cpus have sun as a target vendor
 240         if 'sparc' in cputype:
 241             ostype = 'sun-solaris'
 242     elif ostype.startswith('MINGW'):
 243         # msys' `uname` does not print gcc configuration, but prints msys
 244         # configuration. so we cannot believe `uname -m`:
 245         # msys1 is always i686 and msys2 is always x86_64.
 246         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 247         # MINGW64 on x86_64.
 248         ostype = 'pc-windows-gnu'
 249         cputype = 'i686'
 250         if os.environ.get('MSYSTEM') == 'MINGW64':
 251             cputype = 'x86_64'
 252     elif ostype.startswith('MSYS'):
 253         ostype = 'pc-windows-gnu'
 254     elif ostype.startswith('CYGWIN_NT'):
 255         cputype = 'i686'
 256         if ostype.endswith('WOW64'):
 257             cputype = 'x86_64'
 258         ostype = 'pc-windows-gnu'
 259     elif sys.platform == 'win32':
 260         # Some Windows platforms might have a `uname` command that returns a
 261         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 262         # these cases, fall back to using sys.platform.
 263         return 'x86_64-pc-windows-msvc'
 264     else:
 265         err = "unknown OS type: {}".format(ostype)
 266         sys.exit(err)
 267
 268     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
 269         cputype = subprocess.check_output(
 270               ['uname', '-p']).strip().decode(default_encoding)
 271     cputype_mapper = {
 272         'BePC': 'i686',
 273         'aarch64': 'aarch64',
 274         'amd64': 'x86_64',
 275         'arm64': 'aarch64',
 276         'i386': 'i686',
 277         'i486': 'i686',
 278         'i686': 'i686',
 279         'i786': 'i686',
 280         'm68k': 'm68k',
 281         'powerpc': 'powerpc',
 282         'powerpc64': 'powerpc64',
 283         'powerpc64le': 'powerpc64le',
 284         'ppc': 'powerpc',
 285         'ppc64': 'powerpc64',
 286         'ppc64le': 'powerpc64le',
 287         'riscv64': 'riscv64gc',
 288         's390x': 's390x',
 289         'x64': 'x86_64',
 290         'x86': 'i686',
 291         'x86-64': 'x86_64',
 292         'x86_64': 'x86_64'
 293     }
 294
 295     # Consider the direct transformation first and then the special cases
 296     if cputype in cputype_mapper:
 297         cputype = cputype_mapper[cputype]
 298     elif cputype in {'xscale', 'arm'}:
 299         cputype = 'arm'
 300         if ostype == 'linux-android':
 301             ostype = 'linux-androideabi'
 302         elif ostype == 'unknown-freebsd':
 303             cputype = subprocess.check_output(
 304                 ['uname', '-p']).strip().decode(default_encoding)
 305             ostype = 'unknown-freebsd'
 306     elif cputype == 'armv6l':
 307         cputype = 'arm'
 308         if ostype == 'linux-android':
 309             ostype = 'linux-androideabi'
 310         else:
 311             ostype += 'eabihf'
 312     elif cputype in {'armv7l', 'armv8l'}:
 313         cputype = 'armv7'
 314         if ostype == 'linux-android':
 315             ostype = 'linux-androideabi'
 316         else:
 317             ostype += 'eabihf'
 318     elif cputype == 'mips':
 319         if sys.byteorder == 'big':
 320             cputype = 'mips'
 321         elif sys.byteorder == 'little':
 322             cputype = 'mipsel'
 323         else:
 324             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 325     elif cputype == 'mips64':
 326         if sys.byteorder == 'big':
 327             cputype = 'mips64'
 328         elif sys.byteorder == 'little':
 329             cputype = 'mips64el'
 330         else:
 331             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 332         # only the n64 ABI is supported, indicate it
 333         ostype += 'abi64'
 334     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 335         pass
 336     else:
 337         err = "unknown cpu type: {}".format(cputype)
 338         sys.exit(err)
 339
 340     return "{}-{}".format(cputype, ostype)
 341
 342
 343 @contextlib.contextmanager
 344 def output(filepath):
 345     tmp = filepath + '.tmp'
 346     with open(tmp, 'w') as f:
 347         yield f
 348     try:
 349         if os.path.exists(filepath):
 350             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 351     except OSError:
 352         shutil.copy2(tmp, filepath)
 353         os.remove(tmp)
 354         return
 355     os.rename(tmp, filepath)
 356
 357
 358 class Stage0Toolchain:
 359     def __init__(self, stage0_payload):
 360         self.date = stage0_payload["date"]
 361         self.version = stage0_payload["version"]
 362
 363     def channel(self):
 364         return self.version + "-" + self.date
 365
 366
 367 class RustBuild(object):
 368     """Provide all the methods required to build Rust"""
 369     def __init__(self):
 370         self.checksums_sha256 = {}
 371         self.stage0_compiler = None
 372         self.stage0_rustfmt = None
 373         self._download_url = ''
 374         self.build = ''
 375         self.build_dir = ''
 376         self.clean = False
 377         self.config_toml = ''
 378         self.rust_root = ''
 379         self.use_locked_deps = ''
 380         self.use_vendored_sources = ''
 381         self.verbose = False
 382         self.git_version = None
 383         self.nix_deps_dir = None
 384         self.rustc_commit = None
 385
 386     def download_toolchain(self, stage0=True, rustc_channel=None):
 387         """Fetch the build system for Rust, written in Rust
 388
 389         This method will build a cache directory, then it will fetch the
 390         tarball which has the stage0 compiler used to then bootstrap the Rust
 391         compiler itself.
 392
 393         Each downloaded tarball is extracted, after that, the script
 394         will move all the content to the right place.
 395         """
 396         if rustc_channel is None:
 397             rustc_channel = self.stage0_compiler.version
 398         bin_root = self.bin_root(stage0)
 399
 400         key = self.stage0_compiler.date
 401         if not stage0:
 402             key += str(self.rustc_commit)
 403         if self.rustc(stage0).startswith(bin_root) and \
 404                 (not os.path.exists(self.rustc(stage0)) or
 405                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 406             if os.path.exists(bin_root):
 407                 shutil.rmtree(bin_root)
 408             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 409             filename = "rust-std-{}-{}{}".format(
 410                 rustc_channel, self.build, tarball_suffix)
 411             pattern = "rust-std-{}".format(self.build)
 412             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 413             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 414                                               tarball_suffix)
 415             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 416             # download-rustc doesn't need its own cargo, it can just use beta's.
 417             if stage0:
 418                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 419                                                 tarball_suffix)
 420                 self._download_component_helper(filename, "cargo", tarball_suffix)
 421                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 422             else:
 423                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 424                 self._download_component_helper(
 425                     filename, "rustc-dev", tarball_suffix, stage0
 426                 )
 427
 428             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 429             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 430             lib_dir = "{}/lib".format(bin_root)
 431             for lib in os.listdir(lib_dir):
 432                 if lib.endswith(".so"):
 433                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 434             with output(self.rustc_stamp(stage0)) as rust_stamp:
 435                 rust_stamp.write(key)
 436
 437         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 438             not os.path.exists(self.rustfmt())
 439             or self.program_out_of_date(
 440                 self.rustfmt_stamp(),
 441                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 442             )
 443         ):
 444             if self.stage0_rustfmt is not None:
 445                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 446                 filename = "rustfmt-{}-{}{}".format(
 447                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 448                 )
 449                 self._download_component_helper(
 450                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 451                 )
 452                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 453                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 454                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 455                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 456
 457         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 458         if self.downloading_llvm() and stage0:
 459             # We want the most recent LLVM submodule update to avoid downloading
 460             # LLVM more often than necessary.
 461             #
 462             # This git command finds that commit SHA, looking for bors-authored
 463             # commits that modified src/llvm-project or other relevant version
 464             # stamp files.
 465             #
 466             # This works even in a repository that has not yet initialized
 467             # submodules.
 468             top_level = subprocess.check_output([
 469                 "git", "rev-parse", "--show-toplevel",
 470             ]).decode(sys.getdefaultencoding()).strip()
 471             llvm_sha = subprocess.check_output([
 472                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 473                 "--first-parent", "HEAD",
 474                 "--",
 475                 "{}/src/llvm-project".format(top_level),
 476                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 477                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 478                 "{}/src/version".format(top_level)
 479             ]).decode(sys.getdefaultencoding()).strip()
 480             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 481             llvm_root = self.llvm_root()
 482             llvm_lib = os.path.join(llvm_root, "lib")
 483             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 484                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 485                 for binary in ["llvm-config", "FileCheck"]:
 486                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 487                 for lib in os.listdir(llvm_lib):
 488                     if lib.endswith(".so"):
 489                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 490                 with output(self.llvm_stamp()) as llvm_stamp:
 491                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 492
 493     def downloading_llvm(self):
 494         opt = self.get_toml('download-ci-llvm', 'llvm')
 495         # This is currently all tier 1 targets and tier 2 targets with host tools
 496         # (since others may not have CI artifacts)
 497         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 498         supported_platforms = [
 499             # tier 1
 500             "aarch64-unknown-linux-gnu",
 501             "i686-pc-windows-gnu",
 502             "i686-pc-windows-msvc",
 503             "i686-unknown-linux-gnu",
 504             "x86_64-unknown-linux-gnu",
 505             "x86_64-apple-darwin",
 506             "x86_64-pc-windows-gnu",
 507             "x86_64-pc-windows-msvc",
 508             # tier 2 with host tools
 509             "aarch64-apple-darwin",
 510             "aarch64-pc-windows-msvc",
 511             "aarch64-unknown-linux-musl",
 512             "arm-unknown-linux-gnueabi",
 513             "arm-unknown-linux-gnueabihf",
 514             "armv7-unknown-linux-gnueabihf",
 515             "mips-unknown-linux-gnu",
 516             "mips64-unknown-linux-gnuabi64",
 517             "mips64el-unknown-linux-gnuabi64",
 518             "mipsel-unknown-linux-gnu",
 519             "powerpc-unknown-linux-gnu",
 520             "powerpc64-unknown-linux-gnu",
 521             "powerpc64le-unknown-linux-gnu",
 522             "riscv64gc-unknown-linux-gnu",
 523             "s390x-unknown-linux-gnu",
 524             "x86_64-unknown-freebsd",
 525             "x86_64-unknown-illumos",
 526             "x86_64-unknown-linux-musl",
 527             "x86_64-unknown-netbsd",
 528         ]
 529         return opt == "true" \
 530             or (opt == "if-available" and self.build in supported_platforms)
 531
 532     def _download_component_helper(
 533         self, filename, pattern, tarball_suffix, stage0=True, key=None
 534     ):
 535         if key is None:
 536             if stage0:
 537                 key = self.stage0_compiler.date
 538             else:
 539                 key = self.rustc_commit
 540         cache_dst = os.path.join(self.build_dir, "cache")
 541         rustc_cache = os.path.join(cache_dst, key)
 542         if not os.path.exists(rustc_cache):
 543             os.makedirs(rustc_cache)
 544
 545         if stage0:
 546             base = self._download_url
 547             url = "dist/{}".format(key)
 548         else:
 549             base = "https://ci-artifacts.rust-lang.org"
 550             url = "rustc-builds/{}".format(self.rustc_commit)
 551         tarball = os.path.join(rustc_cache, filename)
 552         if not os.path.exists(tarball):
 553             get(
 554                 base,
 555                 "{}/{}".format(url, filename),
 556                 tarball,
 557                 self.checksums_sha256,
 558                 verbose=self.verbose,
 559                 do_verify=stage0,
 560             )
 561         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 562
 563     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 564         if not llvm_sha:
 565             print("error: could not find commit hash for downloading LLVM")
 566             print("help: maybe your repository history is too shallow?")
 567             print("help: consider disabling `download-ci-llvm`")
 568             print("help: or fetch enough history to include one upstream commit")
 569             exit(1)
 570         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 571         cache_dst = os.path.join(self.build_dir, "cache")
 572         rustc_cache = os.path.join(cache_dst, cache_prefix)
 573         if not os.path.exists(rustc_cache):
 574             os.makedirs(rustc_cache)
 575
 576         base = "https://ci-artifacts.rust-lang.org"
 577         url = "rustc-builds/{}".format(llvm_sha)
 578         if llvm_assertions:
 579             url = url.replace('rustc-builds', 'rustc-builds-alt')
 580         # ci-artifacts are only stored as .xz, not .gz
 581         if not support_xz():
 582             print("error: XZ support is required to download LLVM")
 583             print("help: consider disabling `download-ci-llvm` or using python3")
 584             exit(1)
 585         tarball_suffix = '.tar.xz'
 586         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 587         tarball = os.path.join(rustc_cache, filename)
 588         if not os.path.exists(tarball):
 589             get(
 590                 base,
 591                 "{}/{}".format(url, filename),
 592                 tarball,
 593                 self.checksums_sha256,
 594                 verbose=self.verbose,
 595                 do_verify=False,
 596             )
 597         unpack(tarball, tarball_suffix, self.llvm_root(),
 598                 match="rust-dev",
 599                 verbose=self.verbose)
 600
 601     def fix_bin_or_dylib(self, fname):
 602         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 603         or the RPATH section, to fix the dynamic library search path
 604
 605         This method is only required on NixOS and uses the PatchELF utility to
 606         change the interpreter/RPATH of ELF executables.
 607
 608         Please see https://nixos.org/patchelf.html for more information
 609         """
 610         default_encoding = sys.getdefaultencoding()
 611         try:
 612             ostype = subprocess.check_output(
 613                 ['uname', '-s']).strip().decode(default_encoding)
 614         except subprocess.CalledProcessError:
 615             return
 616         except OSError as reason:
 617             if getattr(reason, 'winerror', None) is not None:
 618                 return
 619             raise reason
 620
 621         if ostype != "Linux":
 622             return
 623
 624         # If the user has asked binaries to be patched for Nix, then
 625         # don't check for NixOS or `/lib`, just continue to the patching.
 626         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 627             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 628             # The latter one does not exist on NixOS when using tmpfs as root.
 629             try:
 630                 with open("/etc/os-release", "r") as f:
 631                     if not any(line.strip() == "ID=nixos" for line in f):
 632                         return
 633             except FileNotFoundError:
 634                 return
 635             if os.path.exists("/lib"):
 636                 return
 637
 638         # At this point we're pretty sure the user is running NixOS or
 639         # using Nix
 640         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 641         print(nix_os_msg, fname)
 642
 643         # Only build `.nix-deps` once.
 644         nix_deps_dir = self.nix_deps_dir
 645         if not nix_deps_dir:
 646             # Run `nix-build` to "build" each dependency (which will likely reuse
 647             # the existing `/nix/store` copy, or at most download a pre-built copy).
 648             #
 649             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 650             # directory, but still reference the actual `/nix/store` path in the rpath
 651             # as it makes it significantly more robust against changes to the location of
 652             # the `.nix-deps` location.
 653             #
 654             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 655             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 656             # patchelf: Needed for patching ELF binaries (see doc comment above).
 657             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 658             nix_expr = '''
 659             with (import <nixpkgs> {});
 660             symlinkJoin {
 661               name = "rust-stage0-dependencies";
 662               paths = [
 663                 zlib
 664                 patchelf
 665                 stdenv.cc.bintools
 666               ];
 667             }
 668             '''
 669             try:
 670                 subprocess.check_output([
 671                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 672                 ])
 673             except subprocess.CalledProcessError as reason:
 674                 print("warning: failed to call nix-build:", reason)
 675                 return
 676             self.nix_deps_dir = nix_deps_dir
 677
 678         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 679         rpath_entries = [
 680             # Relative default, all binary and dynamic libraries we ship
 681             # appear to have this (even when `../lib` is redundant).
 682             "$ORIGIN/../lib",
 683             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 684         ]
 685         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 686         if not fname.endswith(".so"):
 687             # Finally, set the corret .interp for binaries
 688             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 689                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 690
 691         try:
 692             subprocess.check_output([patchelf] + patchelf_args + [fname])
 693         except subprocess.CalledProcessError as reason:
 694             print("warning: failed to call patchelf:", reason)
 695             return
 696
 697     # If `download-rustc` is set, download the most recent commit with CI artifacts
 698     def maybe_download_ci_toolchain(self):
 699         # If `download-rustc` is not set, default to rebuilding.
 700         download_rustc = self.get_toml("download-rustc", section="rust")
 701         if download_rustc is None or download_rustc == "false":
 702             return None
 703         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 704
 705         # Handle running from a directory other than the top level
 706         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 707         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 708         compiler = "{}/compiler/".format(top_level)
 709         library = "{}/library/".format(top_level)
 710
 711         # Look for a version to compare to based on the current commit.
 712         # Only commits merged by bors will have CI artifacts.
 713         merge_base = [
 714             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 715             "--first-parent", "HEAD"
 716         ]
 717         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 718         if not commit:
 719             print("error: could not find commit hash for downloading rustc")
 720             print("help: maybe your repository history is too shallow?")
 721             print("help: consider disabling `download-rustc`")
 722             print("help: or fetch enough history to include one upstream commit")
 723             exit(1)
 724
 725         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 726         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 727         if status != 0:
 728             if download_rustc == "if-unchanged":
 729                 if self.verbose:
 730                     print("warning: saw changes to compiler/ or library/ since {}; " \
 731                           "ignoring `download-rustc`".format(commit))
 732                 return None
 733             print("warning: `download-rustc` is enabled, but there are changes to " \
 734                   "compiler/ or library/")
 735
 736         if self.verbose:
 737             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
 738         self.rustc_commit = commit
 739         # FIXME: support downloading artifacts from the beta channel
 740         self.download_toolchain(False, "nightly")
 741
 742     def rustc_stamp(self, stage0):
 743         """Return the path for .rustc-stamp at the given stage
 744
 745         >>> rb = RustBuild()
 746         >>> rb.build_dir = "build"
 747         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 748         True
 749         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 750         True
 751         """
 752         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 753
 754     def rustfmt_stamp(self):
 755         """Return the path for .rustfmt-stamp
 756
 757         >>> rb = RustBuild()
 758         >>> rb.build_dir = "build"
 759         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 760         True
 761         """
 762         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 763
 764     def llvm_stamp(self):
 765         """Return the path for .rustfmt-stamp
 766
 767         >>> rb = RustBuild()
 768         >>> rb.build_dir = "build"
 769         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 770         True
 771         """
 772         return os.path.join(self.llvm_root(), '.llvm-stamp')
 773
 774
 775     def program_out_of_date(self, stamp_path, key):
 776         """Check if the given program stamp is out of date"""
 777         if not os.path.exists(stamp_path) or self.clean:
 778             return True
 779         with open(stamp_path, 'r') as stamp:
 780             return key != stamp.read()
 781
 782     def bin_root(self, stage0):
 783         """Return the binary root directory for the given stage
 784
 785         >>> rb = RustBuild()
 786         >>> rb.build_dir = "build"
 787         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 788         True
 789         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 790         True
 791
 792         When the 'build' property is given should be a nested directory:
 793
 794         >>> rb.build = "devel"
 795         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 796         True
 797         """
 798         if stage0:
 799             subdir = "stage0"
 800         else:
 801             subdir = "ci-rustc"
 802         return os.path.join(self.build_dir, self.build, subdir)
 803
 804     def llvm_root(self):
 805         """Return the CI LLVM root directory
 806
 807         >>> rb = RustBuild()
 808         >>> rb.build_dir = "build"
 809         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 810         True
 811
 812         When the 'build' property is given should be a nested directory:
 813
 814         >>> rb.build = "devel"
 815         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 816         True
 817         """
 818         return os.path.join(self.build_dir, self.build, "ci-llvm")
 819
 820     def get_toml(self, key, section=None):
 821         """Returns the value of the given key in config.toml, otherwise returns None
 822
 823         >>> rb = RustBuild()
 824         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 825         >>> rb.get_toml("key2")
 826         'value2'
 827
 828         If the key does not exists, the result is None:
 829
 830         >>> rb.get_toml("key3") is None
 831         True
 832
 833         Optionally also matches the section the key appears in
 834
 835         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 836         >>> rb.get_toml('key', 'a')
 837         'value1'
 838         >>> rb.get_toml('key', 'b')
 839         'value2'
 840         >>> rb.get_toml('key', 'c') is None
 841         True
 842
 843         >>> rb.config_toml = 'key1 = true'
 844         >>> rb.get_toml("key1")
 845         'true'
 846         """
 847
 848         cur_section = None
 849         for line in self.config_toml.splitlines():
 850             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 851             if section_match is not None:
 852                 cur_section = section_match.group(1)
 853
 854             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 855             if match is not None:
 856                 value = match.group(1)
 857                 if section is None or section == cur_section:
 858                     return self.get_string(value) or value.strip()
 859         return None
 860
 861     def cargo(self):
 862         """Return config path for cargo"""
 863         return self.program_config('cargo')
 864
 865     def rustc(self, stage0):
 866         """Return config path for rustc"""
 867         return self.program_config('rustc', stage0)
 868
 869     def rustfmt(self):
 870         """Return config path for rustfmt"""
 871         if self.stage0_rustfmt is None:
 872             return None
 873         return self.program_config('rustfmt')
 874
 875     def program_config(self, program, stage0=True):
 876         """Return config path for the given program at the given stage
 877
 878         >>> rb = RustBuild()
 879         >>> rb.config_toml = 'rustc = "rustc"\\n'
 880         >>> rb.program_config('rustc')
 881         'rustc'
 882         >>> rb.config_toml = ''
 883         >>> cargo_path = rb.program_config('cargo', True)
 884         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 885         ... "bin", "cargo")
 886         True
 887         >>> cargo_path = rb.program_config('cargo', False)
 888         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 889         ... "bin", "cargo")
 890         True
 891         """
 892         config = self.get_toml(program)
 893         if config:
 894             return os.path.expanduser(config)
 895         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 896             program, self.exe_suffix()))
 897
 898     @staticmethod
 899     def get_string(line):
 900         """Return the value between double quotes
 901
 902         >>> RustBuild.get_string('    "devel"   ')
 903         'devel'
 904         >>> RustBuild.get_string("    'devel'   ")
 905         'devel'
 906         >>> RustBuild.get_string('devel') is None
 907         True
 908         >>> RustBuild.get_string('    "devel   ')
 909         ''
 910         """
 911         start = line.find('"')
 912         if start != -1:
 913             end = start + 1 + line[start + 1:].find('"')
 914             return line[start + 1:end]
 915         start = line.find('\'')
 916         if start != -1:
 917             end = start + 1 + line[start + 1:].find('\'')
 918             return line[start + 1:end]
 919         return None
 920
 921     @staticmethod
 922     def exe_suffix():
 923         """Return a suffix for executables"""
 924         if sys.platform == 'win32':
 925             return '.exe'
 926         return ''
 927
 928     def bootstrap_binary(self):
 929         """Return the path of the bootstrap binary
 930
 931         >>> rb = RustBuild()
 932         >>> rb.build_dir = "build"
 933         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 934         ... "debug", "bootstrap")
 935         True
 936         """
 937         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 938
 939     def build_bootstrap(self):
 940         """Build bootstrap"""
 941         build_dir = os.path.join(self.build_dir, "bootstrap")
 942         if self.clean and os.path.exists(build_dir):
 943             shutil.rmtree(build_dir)
 944         env = os.environ.copy()
 945         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 946         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 947         if "CARGO_BUILD_TARGET" in env:
 948             del env["CARGO_BUILD_TARGET"]
 949         env["CARGO_TARGET_DIR"] = build_dir
 950         env["RUSTC"] = self.rustc(True)
 951         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 952             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 953             if "LD_LIBRARY_PATH" in env else ""
 954         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 955             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 956             if "DYLD_LIBRARY_PATH" in env else ""
 957         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 958             (os.pathsep + env["LIBRARY_PATH"]) \
 959             if "LIBRARY_PATH" in env else ""
 960
 961         # preserve existing RUSTFLAGS
 962         env.setdefault("RUSTFLAGS", "")
 963         build_section = "target.{}".format(self.build)
 964         target_features = []
 965         if self.get_toml("crt-static", build_section) == "true":
 966             target_features += ["+crt-static"]
 967         elif self.get_toml("crt-static", build_section) == "false":
 968             target_features += ["-crt-static"]
 969         if target_features:
 970             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 971         target_linker = self.get_toml("linker", build_section)
 972         if target_linker is not None:
 973             env["RUSTFLAGS"] += " -C linker=" + target_linker
 974         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 975         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
 976         if self.get_toml("deny-warnings", "rust") != "false":
 977             env["RUSTFLAGS"] += " -Dwarnings"
 978
 979         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
 980             os.pathsep + env["PATH"]
 981         if not os.path.isfile(self.cargo()):
 982             raise Exception("no cargo executable found at `{}`".format(
 983                 self.cargo()))
 984         args = [self.cargo(), "build", "--manifest-path",
 985                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 986         for _ in range(0, self.verbose):
 987             args.append("--verbose")
 988         if self.use_locked_deps:
 989             args.append("--locked")
 990         if self.use_vendored_sources:
 991             args.append("--frozen")
 992         run(args, env=env, verbose=self.verbose)
 993
 994     def build_triple(self):
 995         """Build triple as in LLVM
 996
 997         Note that `default_build_triple` is moderately expensive,
 998         so use `self.build` where possible.
 999         """
1000         config = self.get_toml('build')
1001         if config:
1002             return config
1003         return default_build_triple(self.verbose)
1004
1005     def check_submodule(self, module, slow_submodules):
1006         if not slow_submodules:
1007             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1008                                            cwd=os.path.join(self.rust_root, module),
1009                                            stdout=subprocess.PIPE)
1010             return checked_out
1011         else:
1012             return None
1013
1014     def update_submodule(self, module, checked_out, recorded_submodules):
1015         module_path = os.path.join(self.rust_root, module)
1016
1017         if checked_out is not None:
1018             default_encoding = sys.getdefaultencoding()
1019             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1020             if recorded_submodules[module] == checked_out:
1021                 return
1022
1023         print("Updating submodule", module)
1024
1025         run(["git", "submodule", "-q", "sync", module],
1026             cwd=self.rust_root, verbose=self.verbose)
1027
1028         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1029         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1030             update_args.append("--progress")
1031         update_args.append(module)
1032         try:
1033             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1034         except RuntimeError:
1035             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1036             print('Either stash the changes by running "git stash" within the submodule\'s')
1037             print('directory, reset them by running "git reset --hard", or commit them.')
1038             print("To reset all submodules' changes run", end=" ")
1039             print('"git submodule foreach --recursive git reset --hard".')
1040             raise SystemExit(1)
1041
1042         run(["git", "reset", "-q", "--hard"],
1043             cwd=module_path, verbose=self.verbose)
1044         run(["git", "clean", "-qdfx"],
1045             cwd=module_path, verbose=self.verbose)
1046
1047     def update_submodules(self):
1048         """Update submodules"""
1049         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1050                 self.get_toml('submodules') == "false":
1051             return
1052
1053         default_encoding = sys.getdefaultencoding()
1054
1055         # check the existence and version of 'git' command
1056         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1057         self.git_version = distutils.version.LooseVersion(git_version_str)
1058
1059         slow_submodules = self.get_toml('fast-submodules') == "false"
1060         start_time = time()
1061         if slow_submodules:
1062             print('Unconditionally updating submodules')
1063         else:
1064             print('Updating only changed submodules')
1065         default_encoding = sys.getdefaultencoding()
1066         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1067         # currently requires everything in a workspace to be "locally present" when starting a
1068         # build, and will give a hard error if any Cargo.toml files are missing.
1069         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1070         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1071         #   That will still require cloning the submodules the second you check the standard
1072         #   library, though...
1073         # FIXME: Is there a way to avoid hard-coding the submodules required?
1074         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1075         submodules = [
1076             "src/tools/rust-installer",
1077             "src/tools/cargo",
1078             "src/tools/rls",
1079             "src/tools/miri",
1080             "library/backtrace",
1081             "library/stdarch"
1082         ]
1083         filtered_submodules = []
1084         submodules_names = []
1085         for module in submodules:
1086             check = self.check_submodule(module, slow_submodules)
1087             filtered_submodules.append((module, check))
1088             submodules_names.append(module)
1089         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1090                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1091         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1092         # { filename: hash }
1093         recorded_submodules = {}
1094         for data in recorded:
1095             # [mode, kind, hash, filename]
1096             data = data.split()
1097             recorded_submodules[data[3]] = data[2]
1098         for module in filtered_submodules:
1099             self.update_submodule(module[0], module[1], recorded_submodules)
1100         print("Submodules updated in %.2f seconds" % (time() - start_time))
1101
1102     def set_dist_environment(self, url):
1103         """Set download URL for normal environment"""
1104         if 'RUSTUP_DIST_SERVER' in os.environ:
1105             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1106         else:
1107             self._download_url = url
1108
1109     def check_vendored_status(self):
1110         """Check that vendoring is configured properly"""
1111         vendor_dir = os.path.join(self.rust_root, 'vendor')
1112         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1113             if os.environ.get('USER') != os.environ['SUDO_USER']:
1114                 self.use_vendored_sources = True
1115                 print('info: looks like you are running this command under `sudo`')
1116                 print('      and so in order to preserve your $HOME this will now')
1117                 print('      use vendored sources by default.')
1118                 if not os.path.exists(vendor_dir):
1119                     print('error: vendoring required, but vendor directory does not exist.')
1120                     print('       Run `cargo vendor` without sudo to initialize the '
1121                           'vendor directory.')
1122                     raise Exception("{} not found".format(vendor_dir))
1123
1124         if self.use_vendored_sources:
1125             config = ("[source.crates-io]\n"
1126                       "replace-with = 'vendored-sources'\n"
1127                       "registry = 'https://example.com'\n"
1128                       "\n"
1129                       "[source.vendored-sources]\n"
1130                       "directory = '{}/vendor'\n"
1131                       .format(self.rust_root))
1132             if not os.path.exists('.cargo'):
1133                 os.makedirs('.cargo')
1134                 with output('.cargo/config') as cargo_config:
1135                     cargo_config.write(config)
1136             else:
1137                 print('info: using vendored source, but .cargo/config is already present.')
1138                 print('      Reusing the current configuration file. But you may want to '
1139                       'configure vendoring like this:')
1140                 print(config)
1141         else:
1142             if os.path.exists('.cargo'):
1143                 shutil.rmtree('.cargo')
1144
1145     def ensure_vendored(self):
1146         """Ensure that the vendored sources are available if needed"""
1147         vendor_dir = os.path.join(self.rust_root, 'vendor')
1148         # Note that this does not handle updating the vendored dependencies if
1149         # the rust git repository is updated. Normal development usually does
1150         # not use vendoring, so hopefully this isn't too much of a problem.
1151         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1152             run([
1153                 self.cargo(),
1154                 "vendor",
1155                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1156                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1157             ], verbose=self.verbose, cwd=self.rust_root)
1158
1159
1160 def bootstrap(help_triggered):
1161     """Configure, fetch, build and run the initial bootstrap"""
1162
1163     # If the user is asking for help, let them know that the whole download-and-build
1164     # process has to happen before anything is printed out.
1165     if help_triggered:
1166         print("info: Downloading and building bootstrap before processing --help")
1167         print("      command. See src/bootstrap/README.md for help with common")
1168         print("      commands.")
1169
1170     parser = argparse.ArgumentParser(description='Build rust')
1171     parser.add_argument('--config')
1172     parser.add_argument('--build')
1173     parser.add_argument('--clean', action='store_true')
1174     parser.add_argument('-v', '--verbose', action='count', default=0)
1175
1176     args = [a for a in sys.argv if a != '-h' and a != '--help']
1177     args, _ = parser.parse_known_args(args)
1178
1179     # Configure initial bootstrap
1180     build = RustBuild()
1181     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1182     build.verbose = args.verbose
1183     build.clean = args.clean
1184
1185     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1186     # exists).
1187     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1188     if not toml_path and os.path.exists('config.toml'):
1189         toml_path = 'config.toml'
1190
1191     if toml_path:
1192         if not os.path.exists(toml_path):
1193             toml_path = os.path.join(build.rust_root, toml_path)
1194
1195         with open(toml_path) as config:
1196             build.config_toml = config.read()
1197
1198     profile = build.get_toml('profile')
1199     if profile is not None:
1200         include_file = 'config.{}.toml'.format(profile)
1201         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1202         include_path = os.path.join(include_dir, include_file)
1203         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1204         # specific key, so appending our defaults at the end allows the user to override them
1205         with open(include_path) as included_toml:
1206             build.config_toml += os.linesep + included_toml.read()
1207
1208     config_verbose = build.get_toml('verbose', 'build')
1209     if config_verbose is not None:
1210         build.verbose = max(build.verbose, int(config_verbose))
1211
1212     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1213
1214     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1215
1216     build.check_vendored_status()
1217
1218     build_dir = build.get_toml('build-dir', 'build') or 'build'
1219     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1220
1221     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1222         data = json.load(f)
1223     build.checksums_sha256 = data["checksums_sha256"]
1224     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1225     if data.get("rustfmt") is not None:
1226         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1227
1228     build.set_dist_environment(data["dist_server"])
1229
1230     build.build = args.build or build.build_triple()
1231     build.update_submodules()
1232
1233     # Fetch/build the bootstrap
1234     build.download_toolchain()
1235     # Download the master compiler if `download-rustc` is set
1236     build.maybe_download_ci_toolchain()
1237     sys.stdout.flush()
1238     build.ensure_vendored()
1239     build.build_bootstrap()
1240     sys.stdout.flush()
1241
1242     # Run the bootstrap
1243     args = [build.bootstrap_binary()]
1244     args.extend(sys.argv[1:])
1245     env = os.environ.copy()
1246     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1247     env["BOOTSTRAP_PYTHON"] = sys.executable
1248     env["BUILD_DIR"] = build.build_dir
1249     env["RUSTC_BOOTSTRAP"] = '1'
1250     if toml_path:
1251         env["BOOTSTRAP_CONFIG"] = toml_path
1252     if build.rustc_commit is not None:
1253         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1254     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1255
1256
1257 def main():
1258     """Entry point for the bootstrap process"""
1259     start_time = time()
1260
1261     # x.py help <cmd> ...
1262     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1263         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1264
1265     help_triggered = (
1266         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1267     try:
1268         bootstrap(help_triggered)
1269         if not help_triggered:
1270             print("Build completed successfully in {}".format(
1271                 format_build_time(time() - start_time)))
1272     except (SystemExit, KeyboardInterrupt) as error:
1273         if hasattr(error, 'code') and isinstance(error.code, int):
1274             exit_code = error.code
1275         else:
1276             exit_code = 1
1277             print(error)
1278         if not help_triggered:
1279             print("Build completed unsuccessfully in {}".format(
1280                 format_build_time(time() - start_time)))
1281         sys.exit(exit_code)
1282
1283
1284 if __name__ == '__main__':
1285     main()