src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time
  17
  18 # Acquire a lock on the build directory to make sure that
  19 # we don't cause a race condition while building
  20 # Lock is created in `build_dir/lock.db`
  21 def acquire_lock(build_dir):
  22     try:
  23         import sqlite3
  24
  25         path = os.path.join(build_dir, "lock.db")
  26         try:
  27             con = sqlite3.Connection(path, timeout=0)
  28             curs = con.cursor()
  29             curs.execute("BEGIN EXCLUSIVE")
  30             # The lock is released when the cursor is dropped
  31             return curs
  32         # If the database is busy then lock has already been acquired
  33         # so we wait for the lock.
  34         # We retry every quarter second so that execution is passed back to python
  35         # so that it can handle signals
  36         except sqlite3.OperationalError:
  37             del con
  38             del curs
  39             print("Waiting for lock on build directory")
  40             con = sqlite3.Connection(path, timeout=0.25)
  41             curs = con.cursor()
  42             while True:
  43                 try:
  44                     curs.execute("BEGIN EXCLUSIVE")
  45                 except sqlite3.OperationalError:
  46                     pass
  47             return curs
  48     except ImportError:
  49         print("warning: sqlite3 not available in python, skipping build directory lock")
  50         print("please file an issue on rust-lang/rust")
  51         print("this is not a problem for non-concurrent x.py invocations")
  52         return None
  53
  54 def support_xz():
  55     try:
  56         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  57             temp_path = temp_file.name
  58         with tarfile.open(temp_path, "w:xz"):
  59             pass
  60         return True
  61     except tarfile.CompressionError:
  62         return False
  63
  64 def get(base, url, path, checksums, verbose=False, do_verify=True):
  65     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  66         temp_path = temp_file.name
  67
  68     try:
  69         if do_verify:
  70             if url not in checksums:
  71                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  72             sha256 = checksums[url]
  73             if os.path.exists(path):
  74                 if verify(path, sha256, False):
  75                     if verbose:
  76                         print("using already-download file", path)
  77                     return
  78                 else:
  79                     if verbose:
  80                         print("ignoring already-download file",
  81                             path, "due to failed verification")
  82                     os.unlink(path)
  83         download(temp_path, "{}/{}".format(base, url), True, verbose)
  84         if do_verify and not verify(temp_path, sha256, verbose):
  85             raise RuntimeError("failed verification")
  86         if verbose:
  87             print("moving {} to {}".format(temp_path, path))
  88         shutil.move(temp_path, path)
  89     finally:
  90         if os.path.isfile(temp_path):
  91             if verbose:
  92                 print("removing", temp_path)
  93             os.unlink(temp_path)
  94
  95
  96 def download(path, url, probably_big, verbose):
  97     for _ in range(0, 4):
  98         try:
  99             _download(path, url, probably_big, verbose, True)
 100             return
 101         except RuntimeError:
 102             print("\nspurious failure, trying again")
 103     _download(path, url, probably_big, verbose, False)
 104
 105
 106 def _download(path, url, probably_big, verbose, exception):
 107     if probably_big or verbose:
 108         print("downloading {}".format(url))
 109     # see https://serverfault.com/questions/301128/how-to-download
 110     if sys.platform == 'win32':
 111         run(["PowerShell.exe", "/nologo", "-Command",
 112              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 113              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
 114             verbose=verbose,
 115             exception=exception)
 116     else:
 117         if probably_big or verbose:
 118             option = "-#"
 119         else:
 120             option = "-s"
 121         require(["curl", "--version"])
 122         run(["curl", option,
 123              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
 124              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
 125              "--retry", "3", "-Sf", "-o", path, url],
 126             verbose=verbose,
 127             exception=exception)
 128
 129
 130 def verify(path, expected, verbose):
 131     """Check if the sha256 sum of the given path is valid"""
 132     if verbose:
 133         print("verifying", path)
 134     with open(path, "rb") as source:
 135         found = hashlib.sha256(source.read()).hexdigest()
 136     verified = found == expected
 137     if not verified:
 138         print("invalid checksum:\n"
 139               "    found:    {}\n"
 140               "    expected: {}".format(found, expected))
 141     return verified
 142
 143
 144 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 145     """Unpack the given tarball file"""
 146     print("extracting", tarball)
 147     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 148     with contextlib.closing(tarfile.open(tarball)) as tar:
 149         for member in tar.getnames():
 150             if "/" not in member:
 151                 continue
 152             name = member.replace(fname + "/", "", 1)
 153             if match is not None and not name.startswith(match):
 154                 continue
 155             name = name[len(match) + 1:]
 156
 157             dst_path = os.path.join(dst, name)
 158             if verbose:
 159                 print("  extracting", member)
 160             tar.extract(member, dst)
 161             src_path = os.path.join(dst, member)
 162             if os.path.isdir(src_path) and os.path.exists(dst_path):
 163                 continue
 164             shutil.move(src_path, dst_path)
 165     shutil.rmtree(os.path.join(dst, fname))
 166
 167
 168 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
 169     """Run a child program in a new process"""
 170     if verbose:
 171         print("running: " + ' '.join(args))
 172     sys.stdout.flush()
 173     # Use Popen here instead of call() as it apparently allows powershell on
 174     # Windows to not lock up waiting for input presumably.
 175     ret = subprocess.Popen(args, **kwargs)
 176     code = ret.wait()
 177     if code != 0:
 178         err = "failed to run: " + ' '.join(args)
 179         if verbose or exception:
 180             raise RuntimeError(err)
 181         # For most failures, we definitely do want to print this error, or the user will have no
 182         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 183         # have already printed an error above, so there's no need to print the exact command we're
 184         # running.
 185         if is_bootstrap:
 186             sys.exit(1)
 187         else:
 188             sys.exit(err)
 189
 190
 191 def require(cmd, exit=True):
 192     '''Run a command, returning its output.
 193     On error,
 194         If `exit` is `True`, exit the process.
 195         Otherwise, return None.'''
 196     try:
 197         return subprocess.check_output(cmd).strip()
 198     except (subprocess.CalledProcessError, OSError) as exc:
 199         if not exit:
 200             return None
 201         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 202         print("Please make sure it's installed and in the path.")
 203         sys.exit(1)
 204
 205
 206 def format_build_time(duration):
 207     """Return a nicer format for build time
 208
 209     >>> format_build_time('300')
 210     '0:05:00'
 211     """
 212     return str(datetime.timedelta(seconds=int(duration)))
 213
 214
 215 def default_build_triple(verbose):
 216     """Build triple as in LLVM"""
 217     # If the user already has a host build triple with an existing `rustc`
 218     # install, use their preference. This fixes most issues with Windows builds
 219     # being detected as GNU instead of MSVC.
 220     default_encoding = sys.getdefaultencoding()
 221     try:
 222         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 223                 stderr=subprocess.DEVNULL)
 224         version = version.decode(default_encoding)
 225         host = next(x for x in version.split('\n') if x.startswith("host: "))
 226         triple = host.split("host: ")[1]
 227         if verbose:
 228             print("detected default triple {}".format(triple))
 229         return triple
 230     except Exception as e:
 231         if verbose:
 232             print("rustup not detected: {}".format(e))
 233             print("falling back to auto-detect")
 234
 235     required = sys.platform != 'win32'
 236     ostype = require(["uname", "-s"], exit=required)
 237     cputype = require(['uname', '-m'], exit=required)
 238
 239     # If we do not have `uname`, assume Windows.
 240     if ostype is None or cputype is None:
 241         return 'x86_64-pc-windows-msvc'
 242
 243     ostype = ostype.decode(default_encoding)
 244     cputype = cputype.decode(default_encoding)
 245
 246     # The goal here is to come up with the same triple as LLVM would,
 247     # at least for the subset of platforms we're willing to target.
 248     ostype_mapper = {
 249         'Darwin': 'apple-darwin',
 250         'DragonFly': 'unknown-dragonfly',
 251         'FreeBSD': 'unknown-freebsd',
 252         'Haiku': 'unknown-haiku',
 253         'NetBSD': 'unknown-netbsd',
 254         'OpenBSD': 'unknown-openbsd'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if ostype in ostype_mapper:
 259         ostype = ostype_mapper[ostype]
 260     elif ostype == 'Linux':
 261         os_from_sp = subprocess.check_output(
 262             ['uname', '-o']).strip().decode(default_encoding)
 263         if os_from_sp == 'Android':
 264             ostype = 'linux-android'
 265         else:
 266             ostype = 'unknown-linux-gnu'
 267     elif ostype == 'SunOS':
 268         ostype = 'pc-solaris'
 269         # On Solaris, uname -m will return a machine classification instead
 270         # of a cpu type, so uname -p is recommended instead.  However, the
 271         # output from that option is too generic for our purposes (it will
 272         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 273         # must be used instead.
 274         cputype = require(['isainfo', '-k']).decode(default_encoding)
 275         # sparc cpus have sun as a target vendor
 276         if 'sparc' in cputype:
 277             ostype = 'sun-solaris'
 278     elif ostype.startswith('MINGW'):
 279         # msys' `uname` does not print gcc configuration, but prints msys
 280         # configuration. so we cannot believe `uname -m`:
 281         # msys1 is always i686 and msys2 is always x86_64.
 282         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 283         # MINGW64 on x86_64.
 284         ostype = 'pc-windows-gnu'
 285         cputype = 'i686'
 286         if os.environ.get('MSYSTEM') == 'MINGW64':
 287             cputype = 'x86_64'
 288     elif ostype.startswith('MSYS'):
 289         ostype = 'pc-windows-gnu'
 290     elif ostype.startswith('CYGWIN_NT'):
 291         cputype = 'i686'
 292         if ostype.endswith('WOW64'):
 293             cputype = 'x86_64'
 294         ostype = 'pc-windows-gnu'
 295     elif sys.platform == 'win32':
 296         # Some Windows platforms might have a `uname` command that returns a
 297         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 298         # these cases, fall back to using sys.platform.
 299         return 'x86_64-pc-windows-msvc'
 300     else:
 301         err = "unknown OS type: {}".format(ostype)
 302         sys.exit(err)
 303
 304     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 305         cputype = subprocess.check_output(
 306               ['uname', '-p']).strip().decode(default_encoding)
 307     cputype_mapper = {
 308         'BePC': 'i686',
 309         'aarch64': 'aarch64',
 310         'amd64': 'x86_64',
 311         'arm64': 'aarch64',
 312         'i386': 'i686',
 313         'i486': 'i686',
 314         'i686': 'i686',
 315         'i786': 'i686',
 316         'm68k': 'm68k',
 317         'powerpc': 'powerpc',
 318         'powerpc64': 'powerpc64',
 319         'powerpc64le': 'powerpc64le',
 320         'ppc': 'powerpc',
 321         'ppc64': 'powerpc64',
 322         'ppc64le': 'powerpc64le',
 323         'riscv64': 'riscv64gc',
 324         's390x': 's390x',
 325         'x64': 'x86_64',
 326         'x86': 'i686',
 327         'x86-64': 'x86_64',
 328         'x86_64': 'x86_64'
 329     }
 330
 331     # Consider the direct transformation first and then the special cases
 332     if cputype in cputype_mapper:
 333         cputype = cputype_mapper[cputype]
 334     elif cputype in {'xscale', 'arm'}:
 335         cputype = 'arm'
 336         if ostype == 'linux-android':
 337             ostype = 'linux-androideabi'
 338         elif ostype == 'unknown-freebsd':
 339             cputype = subprocess.check_output(
 340                 ['uname', '-p']).strip().decode(default_encoding)
 341             ostype = 'unknown-freebsd'
 342     elif cputype == 'armv6l':
 343         cputype = 'arm'
 344         if ostype == 'linux-android':
 345             ostype = 'linux-androideabi'
 346         else:
 347             ostype += 'eabihf'
 348     elif cputype in {'armv7l', 'armv8l'}:
 349         cputype = 'armv7'
 350         if ostype == 'linux-android':
 351             ostype = 'linux-androideabi'
 352         else:
 353             ostype += 'eabihf'
 354     elif cputype == 'mips':
 355         if sys.byteorder == 'big':
 356             cputype = 'mips'
 357         elif sys.byteorder == 'little':
 358             cputype = 'mipsel'
 359         else:
 360             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 361     elif cputype == 'mips64':
 362         if sys.byteorder == 'big':
 363             cputype = 'mips64'
 364         elif sys.byteorder == 'little':
 365             cputype = 'mips64el'
 366         else:
 367             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 368         # only the n64 ABI is supported, indicate it
 369         ostype += 'abi64'
 370     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 371         pass
 372     else:
 373         err = "unknown cpu type: {}".format(cputype)
 374         sys.exit(err)
 375
 376     return "{}-{}".format(cputype, ostype)
 377
 378
 379 @contextlib.contextmanager
 380 def output(filepath):
 381     tmp = filepath + '.tmp'
 382     with open(tmp, 'w') as f:
 383         yield f
 384     try:
 385         if os.path.exists(filepath):
 386             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 387     except OSError:
 388         shutil.copy2(tmp, filepath)
 389         os.remove(tmp)
 390         return
 391     os.rename(tmp, filepath)
 392
 393
 394 class Stage0Toolchain:
 395     def __init__(self, stage0_payload):
 396         self.date = stage0_payload["date"]
 397         self.version = stage0_payload["version"]
 398
 399     def channel(self):
 400         return self.version + "-" + self.date
 401
 402
 403 class RustBuild(object):
 404     """Provide all the methods required to build Rust"""
 405     def __init__(self):
 406         self.checksums_sha256 = {}
 407         self.stage0_compiler = None
 408         self.stage0_rustfmt = None
 409         self._download_url = ''
 410         self.build = ''
 411         self.build_dir = ''
 412         self.clean = False
 413         self.config_toml = ''
 414         self.rust_root = ''
 415         self.use_locked_deps = ''
 416         self.use_vendored_sources = ''
 417         self.verbose = False
 418         self.git_version = None
 419         self.nix_deps_dir = None
 420         self.rustc_commit = None
 421
 422     def download_toolchain(self, stage0=True, rustc_channel=None):
 423         """Fetch the build system for Rust, written in Rust
 424
 425         This method will build a cache directory, then it will fetch the
 426         tarball which has the stage0 compiler used to then bootstrap the Rust
 427         compiler itself.
 428
 429         Each downloaded tarball is extracted, after that, the script
 430         will move all the content to the right place.
 431         """
 432         if rustc_channel is None:
 433             rustc_channel = self.stage0_compiler.version
 434         bin_root = self.bin_root(stage0)
 435
 436         key = self.stage0_compiler.date
 437         if not stage0:
 438             key += str(self.rustc_commit)
 439         if self.rustc(stage0).startswith(bin_root) and \
 440                 (not os.path.exists(self.rustc(stage0)) or
 441                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 442             if os.path.exists(bin_root):
 443                 shutil.rmtree(bin_root)
 444             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 445             filename = "rust-std-{}-{}{}".format(
 446                 rustc_channel, self.build, tarball_suffix)
 447             pattern = "rust-std-{}".format(self.build)
 448             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 449             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 450                                               tarball_suffix)
 451             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 452             # download-rustc doesn't need its own cargo, it can just use beta's.
 453             if stage0:
 454                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 455                                                 tarball_suffix)
 456                 self._download_component_helper(filename, "cargo", tarball_suffix)
 457                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 458             else:
 459                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 460                 self._download_component_helper(
 461                     filename, "rustc-dev", tarball_suffix, stage0
 462                 )
 463
 464             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 465             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 466             lib_dir = "{}/lib".format(bin_root)
 467             for lib in os.listdir(lib_dir):
 468                 if lib.endswith(".so"):
 469                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 470             with output(self.rustc_stamp(stage0)) as rust_stamp:
 471                 rust_stamp.write(key)
 472
 473         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 474             not os.path.exists(self.rustfmt())
 475             or self.program_out_of_date(
 476                 self.rustfmt_stamp(),
 477                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 478             )
 479         ):
 480             if self.stage0_rustfmt is not None:
 481                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 482                 filename = "rustfmt-{}-{}{}".format(
 483                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 484                 )
 485                 self._download_component_helper(
 486                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 487                 )
 488                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 489                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 490                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 491                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 492
 493         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 494         if self.downloading_llvm() and stage0:
 495             # We want the most recent LLVM submodule update to avoid downloading
 496             # LLVM more often than necessary.
 497             #
 498             # This git command finds that commit SHA, looking for bors-authored
 499             # commits that modified src/llvm-project or other relevant version
 500             # stamp files.
 501             #
 502             # This works even in a repository that has not yet initialized
 503             # submodules.
 504             top_level = subprocess.check_output([
 505                 "git", "rev-parse", "--show-toplevel",
 506             ]).decode(sys.getdefaultencoding()).strip()
 507             llvm_sha = subprocess.check_output([
 508                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 509                 "--first-parent", "HEAD",
 510                 "--",
 511                 "{}/src/llvm-project".format(top_level),
 512                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 513                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 514                 "{}/src/version".format(top_level)
 515             ]).decode(sys.getdefaultencoding()).strip()
 516             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 517             llvm_root = self.llvm_root()
 518             llvm_lib = os.path.join(llvm_root, "lib")
 519             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 520                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 521                 for binary in ["llvm-config", "FileCheck"]:
 522                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 523                 for lib in os.listdir(llvm_lib):
 524                     if lib.endswith(".so"):
 525                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 526                 with output(self.llvm_stamp()) as llvm_stamp:
 527                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 528
 529     def downloading_llvm(self):
 530         opt = self.get_toml('download-ci-llvm', 'llvm')
 531         # This is currently all tier 1 targets and tier 2 targets with host tools
 532         # (since others may not have CI artifacts)
 533         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 534         supported_platforms = [
 535             # tier 1
 536             "aarch64-unknown-linux-gnu",
 537             "i686-pc-windows-gnu",
 538             "i686-pc-windows-msvc",
 539             "i686-unknown-linux-gnu",
 540             "x86_64-unknown-linux-gnu",
 541             "x86_64-apple-darwin",
 542             "x86_64-pc-windows-gnu",
 543             "x86_64-pc-windows-msvc",
 544             # tier 2 with host tools
 545             "aarch64-apple-darwin",
 546             "aarch64-pc-windows-msvc",
 547             "aarch64-unknown-linux-musl",
 548             "arm-unknown-linux-gnueabi",
 549             "arm-unknown-linux-gnueabihf",
 550             "armv7-unknown-linux-gnueabihf",
 551             "mips-unknown-linux-gnu",
 552             "mips64-unknown-linux-gnuabi64",
 553             "mips64el-unknown-linux-gnuabi64",
 554             "mipsel-unknown-linux-gnu",
 555             "powerpc-unknown-linux-gnu",
 556             "powerpc64-unknown-linux-gnu",
 557             "powerpc64le-unknown-linux-gnu",
 558             "riscv64gc-unknown-linux-gnu",
 559             "s390x-unknown-linux-gnu",
 560             "x86_64-unknown-freebsd",
 561             "x86_64-unknown-illumos",
 562             "x86_64-unknown-linux-musl",
 563             "x86_64-unknown-netbsd",
 564         ]
 565         return opt == "true" \
 566             or (opt == "if-available" and self.build in supported_platforms)
 567
 568     def _download_component_helper(
 569         self, filename, pattern, tarball_suffix, stage0=True, key=None
 570     ):
 571         if key is None:
 572             if stage0:
 573                 key = self.stage0_compiler.date
 574             else:
 575                 key = self.rustc_commit
 576         cache_dst = os.path.join(self.build_dir, "cache")
 577         rustc_cache = os.path.join(cache_dst, key)
 578         if not os.path.exists(rustc_cache):
 579             os.makedirs(rustc_cache)
 580
 581         if stage0:
 582             base = self._download_url
 583             url = "dist/{}".format(key)
 584         else:
 585             base = "https://ci-artifacts.rust-lang.org"
 586             url = "rustc-builds/{}".format(self.rustc_commit)
 587         tarball = os.path.join(rustc_cache, filename)
 588         if not os.path.exists(tarball):
 589             get(
 590                 base,
 591                 "{}/{}".format(url, filename),
 592                 tarball,
 593                 self.checksums_sha256,
 594                 verbose=self.verbose,
 595                 do_verify=stage0,
 596             )
 597         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 598
 599     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 600         if not llvm_sha:
 601             print("error: could not find commit hash for downloading LLVM")
 602             print("help: maybe your repository history is too shallow?")
 603             print("help: consider disabling `download-ci-llvm`")
 604             print("help: or fetch enough history to include one upstream commit")
 605             exit(1)
 606         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 607         cache_dst = os.path.join(self.build_dir, "cache")
 608         rustc_cache = os.path.join(cache_dst, cache_prefix)
 609         if not os.path.exists(rustc_cache):
 610             os.makedirs(rustc_cache)
 611
 612         base = "https://ci-artifacts.rust-lang.org"
 613         url = "rustc-builds/{}".format(llvm_sha)
 614         if llvm_assertions:
 615             url = url.replace('rustc-builds', 'rustc-builds-alt')
 616         # ci-artifacts are only stored as .xz, not .gz
 617         if not support_xz():
 618             print("error: XZ support is required to download LLVM")
 619             print("help: consider disabling `download-ci-llvm` or using python3")
 620             exit(1)
 621         tarball_suffix = '.tar.xz'
 622         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 623         tarball = os.path.join(rustc_cache, filename)
 624         if not os.path.exists(tarball):
 625             get(
 626                 base,
 627                 "{}/{}".format(url, filename),
 628                 tarball,
 629                 self.checksums_sha256,
 630                 verbose=self.verbose,
 631                 do_verify=False,
 632             )
 633         unpack(tarball, tarball_suffix, self.llvm_root(),
 634                 match="rust-dev",
 635                 verbose=self.verbose)
 636
 637     def fix_bin_or_dylib(self, fname):
 638         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 639         or the RPATH section, to fix the dynamic library search path
 640
 641         This method is only required on NixOS and uses the PatchELF utility to
 642         change the interpreter/RPATH of ELF executables.
 643
 644         Please see https://nixos.org/patchelf.html for more information
 645         """
 646         default_encoding = sys.getdefaultencoding()
 647         try:
 648             ostype = subprocess.check_output(
 649                 ['uname', '-s']).strip().decode(default_encoding)
 650         except subprocess.CalledProcessError:
 651             return
 652         except OSError as reason:
 653             if getattr(reason, 'winerror', None) is not None:
 654                 return
 655             raise reason
 656
 657         if ostype != "Linux":
 658             return
 659
 660         # If the user has asked binaries to be patched for Nix, then
 661         # don't check for NixOS or `/lib`, just continue to the patching.
 662         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 663             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 664             # The latter one does not exist on NixOS when using tmpfs as root.
 665             try:
 666                 with open("/etc/os-release", "r") as f:
 667                     if not any(line.strip() == "ID=nixos" for line in f):
 668                         return
 669             except FileNotFoundError:
 670                 return
 671             if os.path.exists("/lib"):
 672                 return
 673
 674         # At this point we're pretty sure the user is running NixOS or
 675         # using Nix
 676         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 677         print(nix_os_msg, fname)
 678
 679         # Only build `.nix-deps` once.
 680         nix_deps_dir = self.nix_deps_dir
 681         if not nix_deps_dir:
 682             # Run `nix-build` to "build" each dependency (which will likely reuse
 683             # the existing `/nix/store` copy, or at most download a pre-built copy).
 684             #
 685             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 686             # directory, but still reference the actual `/nix/store` path in the rpath
 687             # as it makes it significantly more robust against changes to the location of
 688             # the `.nix-deps` location.
 689             #
 690             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 691             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 692             # patchelf: Needed for patching ELF binaries (see doc comment above).
 693             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 694             nix_expr = '''
 695             with (import <nixpkgs> {});
 696             symlinkJoin {
 697               name = "rust-stage0-dependencies";
 698               paths = [
 699                 zlib
 700                 patchelf
 701                 stdenv.cc.bintools
 702               ];
 703             }
 704             '''
 705             try:
 706                 subprocess.check_output([
 707                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 708                 ])
 709             except subprocess.CalledProcessError as reason:
 710                 print("warning: failed to call nix-build:", reason)
 711                 return
 712             self.nix_deps_dir = nix_deps_dir
 713
 714         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 715         rpath_entries = [
 716             # Relative default, all binary and dynamic libraries we ship
 717             # appear to have this (even when `../lib` is redundant).
 718             "$ORIGIN/../lib",
 719             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 720         ]
 721         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 722         if not fname.endswith(".so"):
 723             # Finally, set the corret .interp for binaries
 724             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 725                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 726
 727         try:
 728             subprocess.check_output([patchelf] + patchelf_args + [fname])
 729         except subprocess.CalledProcessError as reason:
 730             print("warning: failed to call patchelf:", reason)
 731             return
 732
 733     # If `download-rustc` is set, download the most recent commit with CI artifacts
 734     def maybe_download_ci_toolchain(self):
 735         # If `download-rustc` is not set, default to rebuilding.
 736         download_rustc = self.get_toml("download-rustc", section="rust")
 737         if download_rustc is None or download_rustc == "false":
 738             return None
 739         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 740
 741         # Handle running from a directory other than the top level
 742         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 743         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 744         compiler = "{}/compiler/".format(top_level)
 745         library = "{}/library/".format(top_level)
 746
 747         # Look for a version to compare to based on the current commit.
 748         # Only commits merged by bors will have CI artifacts.
 749         merge_base = [
 750             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 751             "--first-parent", "HEAD"
 752         ]
 753         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 754         if not commit:
 755             print("error: could not find commit hash for downloading rustc")
 756             print("help: maybe your repository history is too shallow?")
 757             print("help: consider disabling `download-rustc`")
 758             print("help: or fetch enough history to include one upstream commit")
 759             exit(1)
 760
 761         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 762         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 763         if status != 0:
 764             if download_rustc == "if-unchanged":
 765                 return None
 766             print("warning: `download-rustc` is enabled, but there are changes to \
 767                    compiler/ or library/")
 768
 769         if self.verbose:
 770             print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
 771         self.rustc_commit = commit
 772         # FIXME: support downloading artifacts from the beta channel
 773         self.download_toolchain(False, "nightly")
 774
 775     def rustc_stamp(self, stage0):
 776         """Return the path for .rustc-stamp at the given stage
 777
 778         >>> rb = RustBuild()
 779         >>> rb.build_dir = "build"
 780         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 781         True
 782         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 783         True
 784         """
 785         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 786
 787     def rustfmt_stamp(self):
 788         """Return the path for .rustfmt-stamp
 789
 790         >>> rb = RustBuild()
 791         >>> rb.build_dir = "build"
 792         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 793         True
 794         """
 795         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 796
 797     def llvm_stamp(self):
 798         """Return the path for .rustfmt-stamp
 799
 800         >>> rb = RustBuild()
 801         >>> rb.build_dir = "build"
 802         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 803         True
 804         """
 805         return os.path.join(self.llvm_root(), '.llvm-stamp')
 806
 807
 808     def program_out_of_date(self, stamp_path, key):
 809         """Check if the given program stamp is out of date"""
 810         if not os.path.exists(stamp_path) or self.clean:
 811             return True
 812         with open(stamp_path, 'r') as stamp:
 813             return key != stamp.read()
 814
 815     def bin_root(self, stage0):
 816         """Return the binary root directory for the given stage
 817
 818         >>> rb = RustBuild()
 819         >>> rb.build_dir = "build"
 820         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 821         True
 822         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 823         True
 824
 825         When the 'build' property is given should be a nested directory:
 826
 827         >>> rb.build = "devel"
 828         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 829         True
 830         """
 831         if stage0:
 832             subdir = "stage0"
 833         else:
 834             subdir = "ci-rustc"
 835         return os.path.join(self.build_dir, self.build, subdir)
 836
 837     def llvm_root(self):
 838         """Return the CI LLVM root directory
 839
 840         >>> rb = RustBuild()
 841         >>> rb.build_dir = "build"
 842         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 843         True
 844
 845         When the 'build' property is given should be a nested directory:
 846
 847         >>> rb.build = "devel"
 848         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 849         True
 850         """
 851         return os.path.join(self.build_dir, self.build, "ci-llvm")
 852
 853     def get_toml(self, key, section=None):
 854         """Returns the value of the given key in config.toml, otherwise returns None
 855
 856         >>> rb = RustBuild()
 857         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 858         >>> rb.get_toml("key2")
 859         'value2'
 860
 861         If the key does not exists, the result is None:
 862
 863         >>> rb.get_toml("key3") is None
 864         True
 865
 866         Optionally also matches the section the key appears in
 867
 868         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 869         >>> rb.get_toml('key', 'a')
 870         'value1'
 871         >>> rb.get_toml('key', 'b')
 872         'value2'
 873         >>> rb.get_toml('key', 'c') is None
 874         True
 875
 876         >>> rb.config_toml = 'key1 = true'
 877         >>> rb.get_toml("key1")
 878         'true'
 879         """
 880
 881         cur_section = None
 882         for line in self.config_toml.splitlines():
 883             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 884             if section_match is not None:
 885                 cur_section = section_match.group(1)
 886
 887             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 888             if match is not None:
 889                 value = match.group(1)
 890                 if section is None or section == cur_section:
 891                     return self.get_string(value) or value.strip()
 892         return None
 893
 894     def cargo(self):
 895         """Return config path for cargo"""
 896         return self.program_config('cargo')
 897
 898     def rustc(self, stage0):
 899         """Return config path for rustc"""
 900         return self.program_config('rustc', stage0)
 901
 902     def rustfmt(self):
 903         """Return config path for rustfmt"""
 904         if self.stage0_rustfmt is None:
 905             return None
 906         return self.program_config('rustfmt')
 907
 908     def program_config(self, program, stage0=True):
 909         """Return config path for the given program at the given stage
 910
 911         >>> rb = RustBuild()
 912         >>> rb.config_toml = 'rustc = "rustc"\\n'
 913         >>> rb.program_config('rustc')
 914         'rustc'
 915         >>> rb.config_toml = ''
 916         >>> cargo_path = rb.program_config('cargo', True)
 917         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 918         ... "bin", "cargo")
 919         True
 920         >>> cargo_path = rb.program_config('cargo', False)
 921         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 922         ... "bin", "cargo")
 923         True
 924         """
 925         config = self.get_toml(program)
 926         if config:
 927             return os.path.expanduser(config)
 928         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 929             program, self.exe_suffix()))
 930
 931     @staticmethod
 932     def get_string(line):
 933         """Return the value between double quotes
 934
 935         >>> RustBuild.get_string('    "devel"   ')
 936         'devel'
 937         >>> RustBuild.get_string("    'devel'   ")
 938         'devel'
 939         >>> RustBuild.get_string('devel') is None
 940         True
 941         >>> RustBuild.get_string('    "devel   ')
 942         ''
 943         """
 944         start = line.find('"')
 945         if start != -1:
 946             end = start + 1 + line[start + 1:].find('"')
 947             return line[start + 1:end]
 948         start = line.find('\'')
 949         if start != -1:
 950             end = start + 1 + line[start + 1:].find('\'')
 951             return line[start + 1:end]
 952         return None
 953
 954     @staticmethod
 955     def exe_suffix():
 956         """Return a suffix for executables"""
 957         if sys.platform == 'win32':
 958             return '.exe'
 959         return ''
 960
 961     def bootstrap_binary(self):
 962         """Return the path of the bootstrap binary
 963
 964         >>> rb = RustBuild()
 965         >>> rb.build_dir = "build"
 966         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 967         ... "debug", "bootstrap")
 968         True
 969         """
 970         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 971
 972     def build_bootstrap(self):
 973         """Build bootstrap"""
 974         build_dir = os.path.join(self.build_dir, "bootstrap")
 975         if self.clean and os.path.exists(build_dir):
 976             shutil.rmtree(build_dir)
 977         env = os.environ.copy()
 978         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 979         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 980         if "CARGO_BUILD_TARGET" in env:
 981             del env["CARGO_BUILD_TARGET"]
 982         env["CARGO_TARGET_DIR"] = build_dir
 983         env["RUSTC"] = self.rustc(True)
 984         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 985             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 986             if "LD_LIBRARY_PATH" in env else ""
 987         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 988             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 989             if "DYLD_LIBRARY_PATH" in env else ""
 990         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 991             (os.pathsep + env["LIBRARY_PATH"]) \
 992             if "LIBRARY_PATH" in env else ""
 993
 994         # preserve existing RUSTFLAGS
 995         env.setdefault("RUSTFLAGS", "")
 996         build_section = "target.{}".format(self.build)
 997         target_features = []
 998         if self.get_toml("crt-static", build_section) == "true":
 999             target_features += ["+crt-static"]
1000         elif self.get_toml("crt-static", build_section) == "false":
1001             target_features += ["-crt-static"]
1002         if target_features:
1003             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1004         target_linker = self.get_toml("linker", build_section)
1005         if target_linker is not None:
1006             env["RUSTFLAGS"] += " -C linker=" + target_linker
1007         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1008         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1009         if self.get_toml("deny-warnings", "rust") != "false":
1010             env["RUSTFLAGS"] += " -Dwarnings"
1011
1012         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1013             os.pathsep + env["PATH"]
1014         if not os.path.isfile(self.cargo()):
1015             raise Exception("no cargo executable found at `{}`".format(
1016                 self.cargo()))
1017         args = [self.cargo(), "build", "--manifest-path",
1018                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1019         for _ in range(0, self.verbose):
1020             args.append("--verbose")
1021         if self.use_locked_deps:
1022             args.append("--locked")
1023         if self.use_vendored_sources:
1024             args.append("--frozen")
1025         run(args, env=env, verbose=self.verbose)
1026
1027     def build_triple(self):
1028         """Build triple as in LLVM
1029
1030         Note that `default_build_triple` is moderately expensive,
1031         so use `self.build` where possible.
1032         """
1033         config = self.get_toml('build')
1034         if config:
1035             return config
1036         return default_build_triple(self.verbose)
1037
1038     def check_submodule(self, module, slow_submodules):
1039         if not slow_submodules:
1040             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1041                                            cwd=os.path.join(self.rust_root, module),
1042                                            stdout=subprocess.PIPE)
1043             return checked_out
1044         else:
1045             return None
1046
1047     def update_submodule(self, module, checked_out, recorded_submodules):
1048         module_path = os.path.join(self.rust_root, module)
1049
1050         if checked_out is not None:
1051             default_encoding = sys.getdefaultencoding()
1052             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1053             if recorded_submodules[module] == checked_out:
1054                 return
1055
1056         print("Updating submodule", module)
1057
1058         run(["git", "submodule", "-q", "sync", module],
1059             cwd=self.rust_root, verbose=self.verbose)
1060
1061         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1062         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1063             update_args.append("--progress")
1064         update_args.append(module)
1065         try:
1066             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1067         except RuntimeError:
1068             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1069             print('Either stash the changes by running "git stash" within the submodule\'s')
1070             print('directory, reset them by running "git reset --hard", or commit them.')
1071             print("To reset all submodules' changes run", end=" ")
1072             print('"git submodule foreach --recursive git reset --hard".')
1073             raise SystemExit(1)
1074
1075         run(["git", "reset", "-q", "--hard"],
1076             cwd=module_path, verbose=self.verbose)
1077         run(["git", "clean", "-qdfx"],
1078             cwd=module_path, verbose=self.verbose)
1079
1080     def update_submodules(self):
1081         """Update submodules"""
1082         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1083                 self.get_toml('submodules') == "false":
1084             return
1085
1086         default_encoding = sys.getdefaultencoding()
1087
1088         # check the existence and version of 'git' command
1089         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1090         self.git_version = distutils.version.LooseVersion(git_version_str)
1091
1092         slow_submodules = self.get_toml('fast-submodules') == "false"
1093         start_time = time()
1094         if slow_submodules:
1095             print('Unconditionally updating submodules')
1096         else:
1097             print('Updating only changed submodules')
1098         default_encoding = sys.getdefaultencoding()
1099         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1100         # currently requires everything in a workspace to be "locally present" when starting a
1101         # build, and will give a hard error if any Cargo.toml files are missing.
1102         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1103         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1104         #   That will still require cloning the submodules the second you check the standard
1105         #   library, though...
1106         # FIXME: Is there a way to avoid hard-coding the submodules required?
1107         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1108         submodules = [
1109             "src/tools/rust-installer",
1110             "src/tools/cargo",
1111             "src/tools/rls",
1112             "src/tools/miri",
1113             "library/backtrace",
1114             "library/stdarch"
1115         ]
1116         filtered_submodules = []
1117         submodules_names = []
1118         for module in submodules:
1119             check = self.check_submodule(module, slow_submodules)
1120             filtered_submodules.append((module, check))
1121             submodules_names.append(module)
1122         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1123                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1124         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1125         # { filename: hash }
1126         recorded_submodules = {}
1127         for data in recorded:
1128             # [mode, kind, hash, filename]
1129             data = data.split()
1130             recorded_submodules[data[3]] = data[2]
1131         for module in filtered_submodules:
1132             self.update_submodule(module[0], module[1], recorded_submodules)
1133         print("Submodules updated in %.2f seconds" % (time() - start_time))
1134
1135     def set_dist_environment(self, url):
1136         """Set download URL for normal environment"""
1137         if 'RUSTUP_DIST_SERVER' in os.environ:
1138             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1139         else:
1140             self._download_url = url
1141
1142     def check_vendored_status(self):
1143         """Check that vendoring is configured properly"""
1144         vendor_dir = os.path.join(self.rust_root, 'vendor')
1145         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1146             if os.environ.get('USER') != os.environ['SUDO_USER']:
1147                 self.use_vendored_sources = True
1148                 print('info: looks like you are running this command under `sudo`')
1149                 print('      and so in order to preserve your $HOME this will now')
1150                 print('      use vendored sources by default.')
1151                 if not os.path.exists(vendor_dir):
1152                     print('error: vendoring required, but vendor directory does not exist.')
1153                     print('       Run `cargo vendor` without sudo to initialize the '
1154                           'vendor directory.')
1155                     raise Exception("{} not found".format(vendor_dir))
1156
1157         if self.use_vendored_sources:
1158             config = ("[source.crates-io]\n"
1159                       "replace-with = 'vendored-sources'\n"
1160                       "registry = 'https://example.com'\n"
1161                       "\n"
1162                       "[source.vendored-sources]\n"
1163                       "directory = '{}/vendor'\n"
1164                       .format(self.rust_root))
1165             if not os.path.exists('.cargo'):
1166                 os.makedirs('.cargo')
1167                 with output('.cargo/config') as cargo_config:
1168                     cargo_config.write(config)
1169             else:
1170                 print('info: using vendored source, but .cargo/config is already present.')
1171                 print('      Reusing the current configuration file. But you may want to '
1172                       'configure vendoring like this:')
1173                 print(config)
1174         else:
1175             if os.path.exists('.cargo'):
1176                 shutil.rmtree('.cargo')
1177
1178     def ensure_vendored(self):
1179         """Ensure that the vendored sources are available if needed"""
1180         vendor_dir = os.path.join(self.rust_root, 'vendor')
1181         # Note that this does not handle updating the vendored dependencies if
1182         # the rust git repository is updated. Normal development usually does
1183         # not use vendoring, so hopefully this isn't too much of a problem.
1184         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1185             run([
1186                 self.cargo(),
1187                 "vendor",
1188                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1189                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1190             ], verbose=self.verbose, cwd=self.rust_root)
1191
1192
1193 def bootstrap(help_triggered):
1194     """Configure, fetch, build and run the initial bootstrap"""
1195
1196     # If the user is asking for help, let them know that the whole download-and-build
1197     # process has to happen before anything is printed out.
1198     if help_triggered:
1199         print("info: Downloading and building bootstrap before processing --help")
1200         print("      command. See src/bootstrap/README.md for help with common")
1201         print("      commands.")
1202
1203     parser = argparse.ArgumentParser(description='Build rust')
1204     parser.add_argument('--config')
1205     parser.add_argument('--build')
1206     parser.add_argument('--clean', action='store_true')
1207     parser.add_argument('-v', '--verbose', action='count', default=0)
1208
1209     args = [a for a in sys.argv if a != '-h' and a != '--help']
1210     args, _ = parser.parse_known_args(args)
1211
1212     # Configure initial bootstrap
1213     build = RustBuild()
1214     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1215     build.verbose = args.verbose
1216     build.clean = args.clean
1217
1218     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1219     # exists).
1220     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1221     if not toml_path and os.path.exists('config.toml'):
1222         toml_path = 'config.toml'
1223
1224     if toml_path:
1225         if not os.path.exists(toml_path):
1226             toml_path = os.path.join(build.rust_root, toml_path)
1227
1228         with open(toml_path) as config:
1229             build.config_toml = config.read()
1230
1231     profile = build.get_toml('profile')
1232     if profile is not None:
1233         include_file = 'config.{}.toml'.format(profile)
1234         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1235         include_path = os.path.join(include_dir, include_file)
1236         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1237         # specific key, so appending our defaults at the end allows the user to override them
1238         with open(include_path) as included_toml:
1239             build.config_toml += os.linesep + included_toml.read()
1240
1241     config_verbose = build.get_toml('verbose', 'build')
1242     if config_verbose is not None:
1243         build.verbose = max(build.verbose, int(config_verbose))
1244
1245     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1246
1247     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1248
1249     build.check_vendored_status()
1250
1251     build_dir = build.get_toml('build-dir', 'build') or 'build'
1252     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1253
1254     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1255         data = json.load(f)
1256     build.checksums_sha256 = data["checksums_sha256"]
1257     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1258     if data.get("rustfmt") is not None:
1259         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1260
1261     build.set_dist_environment(data["dist_server"])
1262
1263     build.build = args.build or build.build_triple()
1264
1265     # Acquire the lock before doing any build actions
1266     # The lock is released when `lock` is dropped
1267     if not os.path.exists(build.build_dir):
1268         os.makedirs(build.build_dir)
1269     lock = acquire_lock(build.build_dir)
1270     build.update_submodules()
1271
1272     # Fetch/build the bootstrap
1273     build.download_toolchain()
1274     # Download the master compiler if `download-rustc` is set
1275     build.maybe_download_ci_toolchain()
1276     sys.stdout.flush()
1277     build.ensure_vendored()
1278     build.build_bootstrap()
1279     sys.stdout.flush()
1280
1281     # Run the bootstrap
1282     args = [build.bootstrap_binary()]
1283     args.extend(sys.argv[1:])
1284     env = os.environ.copy()
1285     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1286     env["BOOTSTRAP_PYTHON"] = sys.executable
1287     env["BUILD_DIR"] = build.build_dir
1288     env["RUSTC_BOOTSTRAP"] = '1'
1289     if toml_path:
1290         env["BOOTSTRAP_CONFIG"] = toml_path
1291     if build.rustc_commit is not None:
1292         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1293     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1294
1295
1296 def main():
1297     """Entry point for the bootstrap process"""
1298     start_time = time()
1299
1300     # x.py help <cmd> ...
1301     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1302         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1303
1304     help_triggered = (
1305         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1306     try:
1307         bootstrap(help_triggered)
1308         if not help_triggered:
1309             print("Build completed successfully in {}".format(
1310                 format_build_time(time() - start_time)))
1311     except (SystemExit, KeyboardInterrupt) as error:
1312         if hasattr(error, 'code') and isinstance(error.code, int):
1313             exit_code = error.code
1314         else:
1315             exit_code = 1
1316             print(error)
1317         if not help_triggered:
1318             print("Build completed unsuccessfully in {}".format(
1319                 format_build_time(time() - start_time)))
1320         sys.exit(exit_code)
1321
1322
1323 if __name__ == '__main__':
1324     main()