src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time
  17
  18 def support_xz():
  19     try:
  20         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21             temp_path = temp_file.name
  22         with tarfile.open(temp_path, "w:xz"):
  23             pass
  24         return True
  25     except tarfile.CompressionError:
  26         return False
  27
  28 def get(base, url, path, checksums, verbose=False, do_verify=True):
  29     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  30         temp_path = temp_file.name
  31
  32     try:
  33         if do_verify:
  34             if url not in checksums:
  35                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  36             sha256 = checksums[url]
  37             if os.path.exists(path):
  38                 if verify(path, sha256, False):
  39                     if verbose:
  40                         print("using already-download file", path)
  41                     return
  42                 else:
  43                     if verbose:
  44                         print("ignoring already-download file",
  45                             path, "due to failed verification")
  46                     os.unlink(path)
  47         download(temp_path, "{}/{}".format(base, url), True, verbose)
  48         if do_verify and not verify(temp_path, sha256, verbose):
  49             raise RuntimeError("failed verification")
  50         if verbose:
  51             print("moving {} to {}".format(temp_path, path))
  52         shutil.move(temp_path, path)
  53     finally:
  54         if os.path.isfile(temp_path):
  55             if verbose:
  56                 print("removing", temp_path)
  57             os.unlink(temp_path)
  58
  59
  60 def download(path, url, probably_big, verbose):
  61     for _ in range(0, 4):
  62         try:
  63             _download(path, url, probably_big, verbose, True)
  64             return
  65         except RuntimeError:
  66             print("\nspurious failure, trying again")
  67     _download(path, url, probably_big, verbose, False)
  68
  69
  70 def _download(path, url, probably_big, verbose, exception):
  71     if probably_big or verbose:
  72         print("downloading {}".format(url))
  73     # see https://serverfault.com/questions/301128/how-to-download
  74     if sys.platform == 'win32':
  75         run(["PowerShell.exe", "/nologo", "-Command",
  76              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  77              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  78             verbose=verbose,
  79             exception=exception)
  80     else:
  81         if probably_big or verbose:
  82             option = "-#"
  83         else:
  84             option = "-s"
  85         require(["curl", "--version"])
  86         run(["curl", option,
  87              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  88              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  89              "--retry", "3", "-Sf", "-o", path, url],
  90             verbose=verbose,
  91             exception=exception)
  92
  93
  94 def verify(path, expected, verbose):
  95     """Check if the sha256 sum of the given path is valid"""
  96     if verbose:
  97         print("verifying", path)
  98     with open(path, "rb") as source:
  99         found = hashlib.sha256(source.read()).hexdigest()
 100     verified = found == expected
 101     if not verified:
 102         print("invalid checksum:\n"
 103               "    found:    {}\n"
 104               "    expected: {}".format(found, expected))
 105     return verified
 106
 107
 108 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 109     """Unpack the given tarball file"""
 110     print("extracting", tarball)
 111     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 112     with contextlib.closing(tarfile.open(tarball)) as tar:
 113         for member in tar.getnames():
 114             if "/" not in member:
 115                 continue
 116             name = member.replace(fname + "/", "", 1)
 117             if match is not None and not name.startswith(match):
 118                 continue
 119             name = name[len(match) + 1:]
 120
 121             dst_path = os.path.join(dst, name)
 122             if verbose:
 123                 print("  extracting", member)
 124             tar.extract(member, dst)
 125             src_path = os.path.join(dst, member)
 126             if os.path.isdir(src_path) and os.path.exists(dst_path):
 127                 continue
 128             shutil.move(src_path, dst_path)
 129     shutil.rmtree(os.path.join(dst, fname))
 130
 131
 132 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
 133     """Run a child program in a new process"""
 134     if verbose:
 135         print("running: " + ' '.join(args))
 136     sys.stdout.flush()
 137     # Use Popen here instead of call() as it apparently allows powershell on
 138     # Windows to not lock up waiting for input presumably.
 139     ret = subprocess.Popen(args, **kwargs)
 140     code = ret.wait()
 141     if code != 0:
 142         err = "failed to run: " + ' '.join(args)
 143         if verbose or exception:
 144             raise RuntimeError(err)
 145         # For most failures, we definitely do want to print this error, or the user will have no
 146         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 147         # have already printed an error above, so there's no need to print the exact command we're
 148         # running.
 149         if is_bootstrap:
 150             sys.exit(1)
 151         else:
 152             sys.exit(err)
 153
 154
 155 def require(cmd, exit=True):
 156     '''Run a command, returning its output.
 157     On error,
 158         If `exit` is `True`, exit the process.
 159         Otherwise, return None.'''
 160     try:
 161         return subprocess.check_output(cmd).strip()
 162     except (subprocess.CalledProcessError, OSError) as exc:
 163         if not exit:
 164             return None
 165         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 166         print("Please make sure it's installed and in the path.")
 167         sys.exit(1)
 168
 169
 170 def format_build_time(duration):
 171     """Return a nicer format for build time
 172
 173     >>> format_build_time('300')
 174     '0:05:00'
 175     """
 176     return str(datetime.timedelta(seconds=int(duration)))
 177
 178
 179 def default_build_triple(verbose):
 180     """Build triple as in LLVM"""
 181     # If the user already has a host build triple with an existing `rustc`
 182     # install, use their preference. This fixes most issues with Windows builds
 183     # being detected as GNU instead of MSVC.
 184     default_encoding = sys.getdefaultencoding()
 185     try:
 186         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 187                 stderr=subprocess.DEVNULL)
 188         version = version.decode(default_encoding)
 189         host = next(x for x in version.split('\n') if x.startswith("host: "))
 190         triple = host.split("host: ")[1]
 191         if verbose:
 192             print("detected default triple {}".format(triple))
 193         return triple
 194     except Exception as e:
 195         if verbose:
 196             print("rustup not detected: {}".format(e))
 197             print("falling back to auto-detect")
 198
 199     required = sys.platform != 'win32'
 200     ostype = require(["uname", "-s"], exit=required)
 201     cputype = require(['uname', '-m'], exit=required)
 202
 203     # If we do not have `uname`, assume Windows.
 204     if ostype is None or cputype is None:
 205         return 'x86_64-pc-windows-msvc'
 206
 207     ostype = ostype.decode(default_encoding)
 208     cputype = cputype.decode(default_encoding)
 209
 210     # The goal here is to come up with the same triple as LLVM would,
 211     # at least for the subset of platforms we're willing to target.
 212     ostype_mapper = {
 213         'Darwin': 'apple-darwin',
 214         'DragonFly': 'unknown-dragonfly',
 215         'FreeBSD': 'unknown-freebsd',
 216         'Haiku': 'unknown-haiku',
 217         'NetBSD': 'unknown-netbsd',
 218         'OpenBSD': 'unknown-openbsd'
 219     }
 220
 221     # Consider the direct transformation first and then the special cases
 222     if ostype in ostype_mapper:
 223         ostype = ostype_mapper[ostype]
 224     elif ostype == 'Linux':
 225         os_from_sp = subprocess.check_output(
 226             ['uname', '-o']).strip().decode(default_encoding)
 227         if os_from_sp == 'Android':
 228             ostype = 'linux-android'
 229         else:
 230             ostype = 'unknown-linux-gnu'
 231     elif ostype == 'SunOS':
 232         ostype = 'pc-solaris'
 233         # On Solaris, uname -m will return a machine classification instead
 234         # of a cpu type, so uname -p is recommended instead.  However, the
 235         # output from that option is too generic for our purposes (it will
 236         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 237         # must be used instead.
 238         cputype = require(['isainfo', '-k']).decode(default_encoding)
 239         # sparc cpus have sun as a target vendor
 240         if 'sparc' in cputype:
 241             ostype = 'sun-solaris'
 242     elif ostype.startswith('MINGW'):
 243         # msys' `uname` does not print gcc configuration, but prints msys
 244         # configuration. so we cannot believe `uname -m`:
 245         # msys1 is always i686 and msys2 is always x86_64.
 246         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 247         # MINGW64 on x86_64.
 248         ostype = 'pc-windows-gnu'
 249         cputype = 'i686'
 250         if os.environ.get('MSYSTEM') == 'MINGW64':
 251             cputype = 'x86_64'
 252     elif ostype.startswith('MSYS'):
 253         ostype = 'pc-windows-gnu'
 254     elif ostype.startswith('CYGWIN_NT'):
 255         cputype = 'i686'
 256         if ostype.endswith('WOW64'):
 257             cputype = 'x86_64'
 258         ostype = 'pc-windows-gnu'
 259     elif sys.platform == 'win32':
 260         # Some Windows platforms might have a `uname` command that returns a
 261         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 262         # these cases, fall back to using sys.platform.
 263         return 'x86_64-pc-windows-msvc'
 264     else:
 265         err = "unknown OS type: {}".format(ostype)
 266         sys.exit(err)
 267
 268     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 269         cputype = subprocess.check_output(
 270               ['uname', '-p']).strip().decode(default_encoding)
 271     cputype_mapper = {
 272         'BePC': 'i686',
 273         'aarch64': 'aarch64',
 274         'amd64': 'x86_64',
 275         'arm64': 'aarch64',
 276         'i386': 'i686',
 277         'i486': 'i686',
 278         'i686': 'i686',
 279         'i786': 'i686',
 280         'm68k': 'm68k',
 281         'powerpc': 'powerpc',
 282         'powerpc64': 'powerpc64',
 283         'powerpc64le': 'powerpc64le',
 284         'ppc': 'powerpc',
 285         'ppc64': 'powerpc64',
 286         'ppc64le': 'powerpc64le',
 287         'riscv64': 'riscv64gc',
 288         's390x': 's390x',
 289         'x64': 'x86_64',
 290         'x86': 'i686',
 291         'x86-64': 'x86_64',
 292         'x86_64': 'x86_64'
 293     }
 294
 295     # Consider the direct transformation first and then the special cases
 296     if cputype in cputype_mapper:
 297         cputype = cputype_mapper[cputype]
 298     elif cputype in {'xscale', 'arm'}:
 299         cputype = 'arm'
 300         if ostype == 'linux-android':
 301             ostype = 'linux-androideabi'
 302         elif ostype == 'unknown-freebsd':
 303             cputype = subprocess.check_output(
 304                 ['uname', '-p']).strip().decode(default_encoding)
 305             ostype = 'unknown-freebsd'
 306     elif cputype == 'armv6l':
 307         cputype = 'arm'
 308         if ostype == 'linux-android':
 309             ostype = 'linux-androideabi'
 310         else:
 311             ostype += 'eabihf'
 312     elif cputype in {'armv7l', 'armv8l'}:
 313         cputype = 'armv7'
 314         if ostype == 'linux-android':
 315             ostype = 'linux-androideabi'
 316         else:
 317             ostype += 'eabihf'
 318     elif cputype == 'mips':
 319         if sys.byteorder == 'big':
 320             cputype = 'mips'
 321         elif sys.byteorder == 'little':
 322             cputype = 'mipsel'
 323         else:
 324             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 325     elif cputype == 'mips64':
 326         if sys.byteorder == 'big':
 327             cputype = 'mips64'
 328         elif sys.byteorder == 'little':
 329             cputype = 'mips64el'
 330         else:
 331             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 332         # only the n64 ABI is supported, indicate it
 333         ostype += 'abi64'
 334     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 335         pass
 336     else:
 337         err = "unknown cpu type: {}".format(cputype)
 338         sys.exit(err)
 339
 340     return "{}-{}".format(cputype, ostype)
 341
 342
 343 @contextlib.contextmanager
 344 def output(filepath):
 345     tmp = filepath + '.tmp'
 346     with open(tmp, 'w') as f:
 347         yield f
 348     try:
 349         if os.path.exists(filepath):
 350             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 351     except OSError:
 352         shutil.copy2(tmp, filepath)
 353         os.remove(tmp)
 354         return
 355     os.rename(tmp, filepath)
 356
 357
 358 class Stage0Toolchain:
 359     def __init__(self, stage0_payload):
 360         self.date = stage0_payload["date"]
 361         self.version = stage0_payload["version"]
 362
 363     def channel(self):
 364         return self.version + "-" + self.date
 365
 366
 367 class RustBuild(object):
 368     """Provide all the methods required to build Rust"""
 369     def __init__(self):
 370         self.checksums_sha256 = {}
 371         self.stage0_compiler = None
 372         self.stage0_rustfmt = None
 373         self._download_url = ''
 374         self.build = ''
 375         self.build_dir = ''
 376         self.clean = False
 377         self.config_toml = ''
 378         self.rust_root = ''
 379         self.use_locked_deps = ''
 380         self.use_vendored_sources = ''
 381         self.verbose = False
 382         self.git_version = None
 383         self.nix_deps_dir = None
 384         self.rustc_commit = None
 385
 386     def download_toolchain(self, stage0=True, rustc_channel=None):
 387         """Fetch the build system for Rust, written in Rust
 388
 389         This method will build a cache directory, then it will fetch the
 390         tarball which has the stage0 compiler used to then bootstrap the Rust
 391         compiler itself.
 392
 393         Each downloaded tarball is extracted, after that, the script
 394         will move all the content to the right place.
 395         """
 396         if rustc_channel is None:
 397             rustc_channel = self.stage0_compiler.version
 398         bin_root = self.bin_root(stage0)
 399
 400         key = self.stage0_compiler.date
 401         if not stage0:
 402             key += str(self.rustc_commit)
 403         if self.rustc(stage0).startswith(bin_root) and \
 404                 (not os.path.exists(self.rustc(stage0)) or
 405                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 406             if os.path.exists(bin_root):
 407                 shutil.rmtree(bin_root)
 408             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 409             filename = "rust-std-{}-{}{}".format(
 410                 rustc_channel, self.build, tarball_suffix)
 411             pattern = "rust-std-{}".format(self.build)
 412             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 413             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 414                                               tarball_suffix)
 415             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 416             # download-rustc doesn't need its own cargo, it can just use beta's.
 417             if stage0:
 418                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 419                                                 tarball_suffix)
 420                 self._download_component_helper(filename, "cargo", tarball_suffix)
 421                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 422             else:
 423                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 424                 self._download_component_helper(
 425                     filename, "rustc-dev", tarball_suffix, stage0
 426                 )
 427
 428             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 429             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 430             lib_dir = "{}/lib".format(bin_root)
 431             for lib in os.listdir(lib_dir):
 432                 if lib.endswith(".so"):
 433                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 434             with output(self.rustc_stamp(stage0)) as rust_stamp:
 435                 rust_stamp.write(key)
 436
 437         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 438             not os.path.exists(self.rustfmt())
 439             or self.program_out_of_date(
 440                 self.rustfmt_stamp(),
 441                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 442             )
 443         ):
 444             if self.stage0_rustfmt is not None:
 445                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 446                 filename = "rustfmt-{}-{}{}".format(
 447                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 448                 )
 449                 self._download_component_helper(
 450                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 451                 )
 452                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 453                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 454                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 455                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 456
 457         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 458         if self.downloading_llvm() and stage0:
 459             # We want the most recent LLVM submodule update to avoid downloading
 460             # LLVM more often than necessary.
 461             #
 462             # This git command finds that commit SHA, looking for bors-authored
 463             # commits that modified src/llvm-project or other relevant version
 464             # stamp files.
 465             #
 466             # This works even in a repository that has not yet initialized
 467             # submodules.
 468             top_level = subprocess.check_output([
 469                 "git", "rev-parse", "--show-toplevel",
 470             ]).decode(sys.getdefaultencoding()).strip()
 471             llvm_sha = subprocess.check_output([
 472                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 473                 "--first-parent", "HEAD",
 474                 "--",
 475                 "{}/src/llvm-project".format(top_level),
 476                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 477                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 478                 "{}/src/version".format(top_level)
 479             ]).decode(sys.getdefaultencoding()).strip()
 480             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 481             llvm_root = self.llvm_root()
 482             llvm_lib = os.path.join(llvm_root, "lib")
 483             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 484                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 485                 for binary in ["llvm-config", "FileCheck"]:
 486                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 487                 for lib in os.listdir(llvm_lib):
 488                     if lib.endswith(".so"):
 489                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 490                 with output(self.llvm_stamp()) as llvm_stamp:
 491                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 492
 493     def downloading_llvm(self):
 494         opt = self.get_toml('download-ci-llvm', 'llvm')
 495         # This is currently all tier 1 targets and tier 2 targets with host tools
 496         # (since others may not have CI artifacts)
 497         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 498         supported_platforms = [
 499             # tier 1
 500             "aarch64-unknown-linux-gnu",
 501             "i686-pc-windows-gnu",
 502             "i686-pc-windows-msvc",
 503             "i686-unknown-linux-gnu",
 504             "x86_64-unknown-linux-gnu",
 505             "x86_64-apple-darwin",
 506             "x86_64-pc-windows-gnu",
 507             "x86_64-pc-windows-msvc",
 508             # tier 2 with host tools
 509             "aarch64-apple-darwin",
 510             "aarch64-pc-windows-msvc",
 511             "aarch64-unknown-linux-musl",
 512             "arm-unknown-linux-gnueabi",
 513             "arm-unknown-linux-gnueabihf",
 514             "armv7-unknown-linux-gnueabihf",
 515             "mips-unknown-linux-gnu",
 516             "mips64-unknown-linux-gnuabi64",
 517             "mips64el-unknown-linux-gnuabi64",
 518             "mipsel-unknown-linux-gnu",
 519             "powerpc-unknown-linux-gnu",
 520             "powerpc64-unknown-linux-gnu",
 521             "powerpc64le-unknown-linux-gnu",
 522             "riscv64gc-unknown-linux-gnu",
 523             "s390x-unknown-linux-gnu",
 524             "x86_64-unknown-freebsd",
 525             "x86_64-unknown-illumos",
 526             "x86_64-unknown-linux-musl",
 527             "x86_64-unknown-netbsd",
 528         ]
 529         return opt == "true" \
 530             or (opt == "if-available" and self.build in supported_platforms)
 531
 532     def _download_component_helper(
 533         self, filename, pattern, tarball_suffix, stage0=True, key=None
 534     ):
 535         if key is None:
 536             if stage0:
 537                 key = self.stage0_compiler.date
 538             else:
 539                 key = self.rustc_commit
 540         cache_dst = os.path.join(self.build_dir, "cache")
 541         rustc_cache = os.path.join(cache_dst, key)
 542         if not os.path.exists(rustc_cache):
 543             os.makedirs(rustc_cache)
 544
 545         if stage0:
 546             base = self._download_url
 547             url = "dist/{}".format(key)
 548         else:
 549             base = "https://ci-artifacts.rust-lang.org"
 550             url = "rustc-builds/{}".format(self.rustc_commit)
 551         tarball = os.path.join(rustc_cache, filename)
 552         if not os.path.exists(tarball):
 553             get(
 554                 base,
 555                 "{}/{}".format(url, filename),
 556                 tarball,
 557                 self.checksums_sha256,
 558                 verbose=self.verbose,
 559                 do_verify=stage0,
 560             )
 561         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 562
 563     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 564         if not llvm_sha:
 565             print("error: could not find commit hash for downloading LLVM")
 566             print("help: maybe your repository history is too shallow?")
 567             print("help: consider disabling `download-ci-llvm`")
 568             print("help: or fetch enough history to include one upstream commit")
 569             exit(1)
 570         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 571         cache_dst = os.path.join(self.build_dir, "cache")
 572         rustc_cache = os.path.join(cache_dst, cache_prefix)
 573         if not os.path.exists(rustc_cache):
 574             os.makedirs(rustc_cache)
 575
 576         base = "https://ci-artifacts.rust-lang.org"
 577         url = "rustc-builds/{}".format(llvm_sha)
 578         if llvm_assertions:
 579             url = url.replace('rustc-builds', 'rustc-builds-alt')
 580         # ci-artifacts are only stored as .xz, not .gz
 581         if not support_xz():
 582             print("error: XZ support is required to download LLVM")
 583             print("help: consider disabling `download-ci-llvm` or using python3")
 584             exit(1)
 585         tarball_suffix = '.tar.xz'
 586         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 587         tarball = os.path.join(rustc_cache, filename)
 588         if not os.path.exists(tarball):
 589             get(
 590                 base,
 591                 "{}/{}".format(url, filename),
 592                 tarball,
 593                 self.checksums_sha256,
 594                 verbose=self.verbose,
 595                 do_verify=False,
 596             )
 597         unpack(tarball, tarball_suffix, self.llvm_root(),
 598                 match="rust-dev",
 599                 verbose=self.verbose)
 600
 601     def fix_bin_or_dylib(self, fname):
 602         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 603         or the RPATH section, to fix the dynamic library search path
 604
 605         This method is only required on NixOS and uses the PatchELF utility to
 606         change the interpreter/RPATH of ELF executables.
 607
 608         Please see https://nixos.org/patchelf.html for more information
 609         """
 610         default_encoding = sys.getdefaultencoding()
 611         try:
 612             ostype = subprocess.check_output(
 613                 ['uname', '-s']).strip().decode(default_encoding)
 614         except subprocess.CalledProcessError:
 615             return
 616         except OSError as reason:
 617             if getattr(reason, 'winerror', None) is not None:
 618                 return
 619             raise reason
 620
 621         if ostype != "Linux":
 622             return
 623
 624         # If the user has asked binaries to be patched for Nix, then
 625         # don't check for NixOS or `/lib`, just continue to the patching.
 626         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 627             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 628             # The latter one does not exist on NixOS when using tmpfs as root.
 629             try:
 630                 with open("/etc/os-release", "r") as f:
 631                     if not any(line.strip() == "ID=nixos" for line in f):
 632                         return
 633             except FileNotFoundError:
 634                 return
 635             if os.path.exists("/lib"):
 636                 return
 637
 638         # At this point we're pretty sure the user is running NixOS or
 639         # using Nix
 640         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 641         print(nix_os_msg, fname)
 642
 643         # Only build `.nix-deps` once.
 644         nix_deps_dir = self.nix_deps_dir
 645         if not nix_deps_dir:
 646             # Run `nix-build` to "build" each dependency (which will likely reuse
 647             # the existing `/nix/store` copy, or at most download a pre-built copy).
 648             #
 649             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 650             # directory, but still reference the actual `/nix/store` path in the rpath
 651             # as it makes it significantly more robust against changes to the location of
 652             # the `.nix-deps` location.
 653             #
 654             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 655             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 656             # patchelf: Needed for patching ELF binaries (see doc comment above).
 657             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 658             nix_expr = '''
 659             with (import <nixpkgs> {});
 660             symlinkJoin {
 661               name = "rust-stage0-dependencies";
 662               paths = [
 663                 zlib
 664                 patchelf
 665                 stdenv.cc.bintools
 666               ];
 667             }
 668             '''
 669             try:
 670                 subprocess.check_output([
 671                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 672                 ])
 673             except subprocess.CalledProcessError as reason:
 674                 print("warning: failed to call nix-build:", reason)
 675                 return
 676             self.nix_deps_dir = nix_deps_dir
 677
 678         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 679         rpath_entries = [
 680             # Relative default, all binary and dynamic libraries we ship
 681             # appear to have this (even when `../lib` is redundant).
 682             "$ORIGIN/../lib",
 683             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 684         ]
 685         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 686         if not fname.endswith(".so"):
 687             # Finally, set the corret .interp for binaries
 688             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 689                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 690
 691         try:
 692             subprocess.check_output([patchelf] + patchelf_args + [fname])
 693         except subprocess.CalledProcessError as reason:
 694             print("warning: failed to call patchelf:", reason)
 695             return
 696
 697     # If `download-rustc` is set, download the most recent commit with CI artifacts
 698     def maybe_download_ci_toolchain(self):
 699         # If `download-rustc` is not set, default to rebuilding.
 700         download_rustc = self.get_toml("download-rustc", section="rust")
 701         if download_rustc is None or download_rustc == "false":
 702             return None
 703         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 704
 705         # Handle running from a directory other than the top level
 706         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 707         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 708         compiler = "{}/compiler/".format(top_level)
 709         library = "{}/library/".format(top_level)
 710
 711         # Look for a version to compare to based on the current commit.
 712         # Only commits merged by bors will have CI artifacts.
 713         merge_base = [
 714             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 715             "--first-parent", "HEAD"
 716         ]
 717         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 718         if not commit:
 719             print("error: could not find commit hash for downloading rustc")
 720             print("help: maybe your repository history is too shallow?")
 721             print("help: consider disabling `download-rustc`")
 722             print("help: or fetch enough history to include one upstream commit")
 723             exit(1)
 724
 725         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 726         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 727         if status != 0:
 728             if download_rustc == "if-unchanged":
 729                 return None
 730             print("warning: `download-rustc` is enabled, but there are changes to \
 731                    compiler/ or library/")
 732
 733         if self.verbose:
 734             print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
 735         self.rustc_commit = commit
 736         # FIXME: support downloading artifacts from the beta channel
 737         self.download_toolchain(False, "nightly")
 738
 739     def rustc_stamp(self, stage0):
 740         """Return the path for .rustc-stamp at the given stage
 741
 742         >>> rb = RustBuild()
 743         >>> rb.build_dir = "build"
 744         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 745         True
 746         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 747         True
 748         """
 749         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 750
 751     def rustfmt_stamp(self):
 752         """Return the path for .rustfmt-stamp
 753
 754         >>> rb = RustBuild()
 755         >>> rb.build_dir = "build"
 756         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 757         True
 758         """
 759         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 760
 761     def llvm_stamp(self):
 762         """Return the path for .rustfmt-stamp
 763
 764         >>> rb = RustBuild()
 765         >>> rb.build_dir = "build"
 766         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 767         True
 768         """
 769         return os.path.join(self.llvm_root(), '.llvm-stamp')
 770
 771
 772     def program_out_of_date(self, stamp_path, key):
 773         """Check if the given program stamp is out of date"""
 774         if not os.path.exists(stamp_path) or self.clean:
 775             return True
 776         with open(stamp_path, 'r') as stamp:
 777             return key != stamp.read()
 778
 779     def bin_root(self, stage0):
 780         """Return the binary root directory for the given stage
 781
 782         >>> rb = RustBuild()
 783         >>> rb.build_dir = "build"
 784         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 785         True
 786         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 787         True
 788
 789         When the 'build' property is given should be a nested directory:
 790
 791         >>> rb.build = "devel"
 792         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 793         True
 794         """
 795         if stage0:
 796             subdir = "stage0"
 797         else:
 798             subdir = "ci-rustc"
 799         return os.path.join(self.build_dir, self.build, subdir)
 800
 801     def llvm_root(self):
 802         """Return the CI LLVM root directory
 803
 804         >>> rb = RustBuild()
 805         >>> rb.build_dir = "build"
 806         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 807         True
 808
 809         When the 'build' property is given should be a nested directory:
 810
 811         >>> rb.build = "devel"
 812         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 813         True
 814         """
 815         return os.path.join(self.build_dir, self.build, "ci-llvm")
 816
 817     def get_toml(self, key, section=None):
 818         """Returns the value of the given key in config.toml, otherwise returns None
 819
 820         >>> rb = RustBuild()
 821         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 822         >>> rb.get_toml("key2")
 823         'value2'
 824
 825         If the key does not exists, the result is None:
 826
 827         >>> rb.get_toml("key3") is None
 828         True
 829
 830         Optionally also matches the section the key appears in
 831
 832         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 833         >>> rb.get_toml('key', 'a')
 834         'value1'
 835         >>> rb.get_toml('key', 'b')
 836         'value2'
 837         >>> rb.get_toml('key', 'c') is None
 838         True
 839
 840         >>> rb.config_toml = 'key1 = true'
 841         >>> rb.get_toml("key1")
 842         'true'
 843         """
 844
 845         cur_section = None
 846         for line in self.config_toml.splitlines():
 847             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 848             if section_match is not None:
 849                 cur_section = section_match.group(1)
 850
 851             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 852             if match is not None:
 853                 value = match.group(1)
 854                 if section is None or section == cur_section:
 855                     return self.get_string(value) or value.strip()
 856         return None
 857
 858     def cargo(self):
 859         """Return config path for cargo"""
 860         return self.program_config('cargo')
 861
 862     def rustc(self, stage0):
 863         """Return config path for rustc"""
 864         return self.program_config('rustc', stage0)
 865
 866     def rustfmt(self):
 867         """Return config path for rustfmt"""
 868         if self.stage0_rustfmt is None:
 869             return None
 870         return self.program_config('rustfmt')
 871
 872     def program_config(self, program, stage0=True):
 873         """Return config path for the given program at the given stage
 874
 875         >>> rb = RustBuild()
 876         >>> rb.config_toml = 'rustc = "rustc"\\n'
 877         >>> rb.program_config('rustc')
 878         'rustc'
 879         >>> rb.config_toml = ''
 880         >>> cargo_path = rb.program_config('cargo', True)
 881         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 882         ... "bin", "cargo")
 883         True
 884         >>> cargo_path = rb.program_config('cargo', False)
 885         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 886         ... "bin", "cargo")
 887         True
 888         """
 889         config = self.get_toml(program)
 890         if config:
 891             return os.path.expanduser(config)
 892         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 893             program, self.exe_suffix()))
 894
 895     @staticmethod
 896     def get_string(line):
 897         """Return the value between double quotes
 898
 899         >>> RustBuild.get_string('    "devel"   ')
 900         'devel'
 901         >>> RustBuild.get_string("    'devel'   ")
 902         'devel'
 903         >>> RustBuild.get_string('devel') is None
 904         True
 905         >>> RustBuild.get_string('    "devel   ')
 906         ''
 907         """
 908         start = line.find('"')
 909         if start != -1:
 910             end = start + 1 + line[start + 1:].find('"')
 911             return line[start + 1:end]
 912         start = line.find('\'')
 913         if start != -1:
 914             end = start + 1 + line[start + 1:].find('\'')
 915             return line[start + 1:end]
 916         return None
 917
 918     @staticmethod
 919     def exe_suffix():
 920         """Return a suffix for executables"""
 921         if sys.platform == 'win32':
 922             return '.exe'
 923         return ''
 924
 925     def bootstrap_binary(self):
 926         """Return the path of the bootstrap binary
 927
 928         >>> rb = RustBuild()
 929         >>> rb.build_dir = "build"
 930         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 931         ... "debug", "bootstrap")
 932         True
 933         """
 934         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 935
 936     def build_bootstrap(self):
 937         """Build bootstrap"""
 938         build_dir = os.path.join(self.build_dir, "bootstrap")
 939         if self.clean and os.path.exists(build_dir):
 940             shutil.rmtree(build_dir)
 941         env = os.environ.copy()
 942         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 943         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 944         if "CARGO_BUILD_TARGET" in env:
 945             del env["CARGO_BUILD_TARGET"]
 946         env["CARGO_TARGET_DIR"] = build_dir
 947         env["RUSTC"] = self.rustc(True)
 948         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 949             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 950             if "LD_LIBRARY_PATH" in env else ""
 951         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 952             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 953             if "DYLD_LIBRARY_PATH" in env else ""
 954         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 955             (os.pathsep + env["LIBRARY_PATH"]) \
 956             if "LIBRARY_PATH" in env else ""
 957
 958         # preserve existing RUSTFLAGS
 959         env.setdefault("RUSTFLAGS", "")
 960         build_section = "target.{}".format(self.build)
 961         target_features = []
 962         if self.get_toml("crt-static", build_section) == "true":
 963             target_features += ["+crt-static"]
 964         elif self.get_toml("crt-static", build_section) == "false":
 965             target_features += ["-crt-static"]
 966         if target_features:
 967             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 968         target_linker = self.get_toml("linker", build_section)
 969         if target_linker is not None:
 970             env["RUSTFLAGS"] += " -C linker=" + target_linker
 971         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 972         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
 973         if self.get_toml("deny-warnings", "rust") != "false":
 974             env["RUSTFLAGS"] += " -Dwarnings"
 975
 976         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
 977             os.pathsep + env["PATH"]
 978         if not os.path.isfile(self.cargo()):
 979             raise Exception("no cargo executable found at `{}`".format(
 980                 self.cargo()))
 981         args = [self.cargo(), "build", "--manifest-path",
 982                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 983         for _ in range(0, self.verbose):
 984             args.append("--verbose")
 985         if self.use_locked_deps:
 986             args.append("--locked")
 987         if self.use_vendored_sources:
 988             args.append("--frozen")
 989         run(args, env=env, verbose=self.verbose)
 990
 991     def build_triple(self):
 992         """Build triple as in LLVM
 993
 994         Note that `default_build_triple` is moderately expensive,
 995         so use `self.build` where possible.
 996         """
 997         config = self.get_toml('build')
 998         if config:
 999             return config
1000         return default_build_triple(self.verbose)
1001
1002     def check_submodule(self, module, slow_submodules):
1003         if not slow_submodules:
1004             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1005                                            cwd=os.path.join(self.rust_root, module),
1006                                            stdout=subprocess.PIPE)
1007             return checked_out
1008         else:
1009             return None
1010
1011     def update_submodule(self, module, checked_out, recorded_submodules):
1012         module_path = os.path.join(self.rust_root, module)
1013
1014         if checked_out is not None:
1015             default_encoding = sys.getdefaultencoding()
1016             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1017             if recorded_submodules[module] == checked_out:
1018                 return
1019
1020         print("Updating submodule", module)
1021
1022         run(["git", "submodule", "-q", "sync", module],
1023             cwd=self.rust_root, verbose=self.verbose)
1024
1025         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1026         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1027             update_args.append("--progress")
1028         update_args.append(module)
1029         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1030
1031         run(["git", "reset", "-q", "--hard"],
1032             cwd=module_path, verbose=self.verbose)
1033         run(["git", "clean", "-qdfx"],
1034             cwd=module_path, verbose=self.verbose)
1035
1036     def update_submodules(self):
1037         """Update submodules"""
1038         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1039                 self.get_toml('submodules') == "false":
1040             return
1041
1042         default_encoding = sys.getdefaultencoding()
1043
1044         # check the existence and version of 'git' command
1045         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1046         self.git_version = distutils.version.LooseVersion(git_version_str)
1047
1048         slow_submodules = self.get_toml('fast-submodules') == "false"
1049         start_time = time()
1050         if slow_submodules:
1051             print('Unconditionally updating submodules')
1052         else:
1053             print('Updating only changed submodules')
1054         default_encoding = sys.getdefaultencoding()
1055         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1056         # currently requires everything in a workspace to be "locally present" when starting a
1057         # build, and will give a hard error if any Cargo.toml files are missing.
1058         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1059         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1060         #   That will still require cloning the submodules the second you check the standard
1061         #   library, though...
1062         # FIXME: Is there a way to avoid hard-coding the submodules required?
1063         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1064         submodules = [
1065             "src/tools/rust-installer",
1066             "src/tools/cargo",
1067             "src/tools/rls",
1068             "src/tools/miri",
1069             "library/backtrace",
1070             "library/stdarch"
1071         ]
1072         filtered_submodules = []
1073         submodules_names = []
1074         for module in submodules:
1075             check = self.check_submodule(module, slow_submodules)
1076             filtered_submodules.append((module, check))
1077             submodules_names.append(module)
1078         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1079                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1080         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1081         # { filename: hash }
1082         recorded_submodules = {}
1083         for data in recorded:
1084             # [mode, kind, hash, filename]
1085             data = data.split()
1086             recorded_submodules[data[3]] = data[2]
1087         for module in filtered_submodules:
1088             self.update_submodule(module[0], module[1], recorded_submodules)
1089         print("Submodules updated in %.2f seconds" % (time() - start_time))
1090
1091     def set_dist_environment(self, url):
1092         """Set download URL for normal environment"""
1093         if 'RUSTUP_DIST_SERVER' in os.environ:
1094             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1095         else:
1096             self._download_url = url
1097
1098     def check_vendored_status(self):
1099         """Check that vendoring is configured properly"""
1100         vendor_dir = os.path.join(self.rust_root, 'vendor')
1101         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1102             if os.environ.get('USER') != os.environ['SUDO_USER']:
1103                 self.use_vendored_sources = True
1104                 print('info: looks like you are running this command under `sudo`')
1105                 print('      and so in order to preserve your $HOME this will now')
1106                 print('      use vendored sources by default.')
1107                 if not os.path.exists(vendor_dir):
1108                     print('error: vendoring required, but vendor directory does not exist.')
1109                     print('       Run `cargo vendor` without sudo to initialize the '
1110                           'vendor directory.')
1111                     raise Exception("{} not found".format(vendor_dir))
1112
1113         if self.use_vendored_sources:
1114             if not os.path.exists('.cargo'):
1115                 os.makedirs('.cargo')
1116             with output('.cargo/config') as cargo_config:
1117                 cargo_config.write(
1118                     "[source.crates-io]\n"
1119                     "replace-with = 'vendored-sources'\n"
1120                     "registry = 'https://example.com'\n"
1121                     "\n"
1122                     "[source.vendored-sources]\n"
1123                     "directory = '{}/vendor'\n"
1124                     .format(self.rust_root))
1125         else:
1126             if os.path.exists('.cargo'):
1127                 shutil.rmtree('.cargo')
1128
1129     def ensure_vendored(self):
1130         """Ensure that the vendored sources are available if needed"""
1131         vendor_dir = os.path.join(self.rust_root, 'vendor')
1132         # Note that this does not handle updating the vendored dependencies if
1133         # the rust git repository is updated. Normal development usually does
1134         # not use vendoring, so hopefully this isn't too much of a problem.
1135         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1136             run([
1137                 self.cargo(),
1138                 "vendor",
1139                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1140                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1141             ], verbose=self.verbose, cwd=self.rust_root)
1142
1143
1144 def bootstrap(help_triggered):
1145     """Configure, fetch, build and run the initial bootstrap"""
1146
1147     # If the user is asking for help, let them know that the whole download-and-build
1148     # process has to happen before anything is printed out.
1149     if help_triggered:
1150         print("info: Downloading and building bootstrap before processing --help")
1151         print("      command. See src/bootstrap/README.md for help with common")
1152         print("      commands.")
1153
1154     parser = argparse.ArgumentParser(description='Build rust')
1155     parser.add_argument('--config')
1156     parser.add_argument('--build')
1157     parser.add_argument('--clean', action='store_true')
1158     parser.add_argument('-v', '--verbose', action='count', default=0)
1159
1160     args = [a for a in sys.argv if a != '-h' and a != '--help']
1161     args, _ = parser.parse_known_args(args)
1162
1163     # Configure initial bootstrap
1164     build = RustBuild()
1165     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1166     build.verbose = args.verbose
1167     build.clean = args.clean
1168
1169     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1170     # exists).
1171     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1172     if not toml_path and os.path.exists('config.toml'):
1173         toml_path = 'config.toml'
1174
1175     if toml_path:
1176         if not os.path.exists(toml_path):
1177             toml_path = os.path.join(build.rust_root, toml_path)
1178
1179         with open(toml_path) as config:
1180             build.config_toml = config.read()
1181
1182     profile = build.get_toml('profile')
1183     if profile is not None:
1184         include_file = 'config.{}.toml'.format(profile)
1185         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1186         include_path = os.path.join(include_dir, include_file)
1187         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1188         # specific key, so appending our defaults at the end allows the user to override them
1189         with open(include_path) as included_toml:
1190             build.config_toml += os.linesep + included_toml.read()
1191
1192     config_verbose = build.get_toml('verbose', 'build')
1193     if config_verbose is not None:
1194         build.verbose = max(build.verbose, int(config_verbose))
1195
1196     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1197
1198     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1199
1200     build.check_vendored_status()
1201
1202     build_dir = build.get_toml('build-dir', 'build') or 'build'
1203     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1204
1205     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1206         data = json.load(f)
1207     build.checksums_sha256 = data["checksums_sha256"]
1208     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1209     if data.get("rustfmt") is not None:
1210         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1211
1212     build.set_dist_environment(data["dist_server"])
1213
1214     build.build = args.build or build.build_triple()
1215     build.update_submodules()
1216
1217     # Fetch/build the bootstrap
1218     build.download_toolchain()
1219     # Download the master compiler if `download-rustc` is set
1220     build.maybe_download_ci_toolchain()
1221     sys.stdout.flush()
1222     build.ensure_vendored()
1223     build.build_bootstrap()
1224     sys.stdout.flush()
1225
1226     # Run the bootstrap
1227     args = [build.bootstrap_binary()]
1228     args.extend(sys.argv[1:])
1229     env = os.environ.copy()
1230     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1231     env["BOOTSTRAP_PYTHON"] = sys.executable
1232     env["BUILD_DIR"] = build.build_dir
1233     env["RUSTC_BOOTSTRAP"] = '1'
1234     if toml_path:
1235         env["BOOTSTRAP_CONFIG"] = toml_path
1236     if build.rustc_commit is not None:
1237         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1238     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1239
1240
1241 def main():
1242     """Entry point for the bootstrap process"""
1243     start_time = time()
1244
1245     # x.py help <cmd> ...
1246     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1247         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1248
1249     help_triggered = (
1250         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1251     try:
1252         bootstrap(help_triggered)
1253         if not help_triggered:
1254             print("Build completed successfully in {}".format(
1255                 format_build_time(time() - start_time)))
1256     except (SystemExit, KeyboardInterrupt) as error:
1257         if hasattr(error, 'code') and isinstance(error.code, int):
1258             exit_code = error.code
1259         else:
1260             exit_code = 1
1261             print(error)
1262         if not help_triggered:
1263             print("Build completed unsuccessfully in {}".format(
1264                 format_build_time(time() - start_time)))
1265         sys.exit(exit_code)
1266
1267
1268 if __name__ == '__main__':
1269     main()