src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time
  17
  18 # Acquire a lock on the build directory to make sure that
  19 # we don't cause a race condition while building
  20 # Lock is created in `build_dir/lock.db`
  21 def acquire_lock(build_dir):
  22     try:
  23         import sqlite3
  24
  25         path = os.path.join(build_dir, "lock.db")
  26         try:
  27             con = sqlite3.Connection(path, timeout=0)
  28             curs = con.cursor()
  29             curs.execute("BEGIN EXCLUSIVE")
  30             # The lock is released when the cursor is dropped
  31             return curs
  32         # If the database is busy then lock has already been acquired
  33         # so we wait for the lock.
  34         # We retry every quarter second so that execution is passed back to python
  35         # so that it can handle signals
  36         except sqlite3.OperationalError:
  37             del con
  38             del curs
  39             print("Waiting for lock on build directory")
  40             con = sqlite3.Connection(path, timeout=0.25)
  41             curs = con.cursor()
  42             while True:
  43                 try:
  44                     curs.execute("BEGIN EXCLUSIVE")
  45                 except sqlite3.OperationalError:
  46                     pass
  47             return curs
  48     except ImportError:
  49         print("warning: sqlite3 not available in python, skipping build directory lock")
  50         print("please file an issue on rust-lang/rust")
  51         print("this is not a problem for non-concurrent x.py invocations")
  52         return None
  53
  54 def support_xz():
  55     try:
  56         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  57             temp_path = temp_file.name
  58         with tarfile.open(temp_path, "w:xz"):
  59             pass
  60         return True
  61     except tarfile.CompressionError:
  62         return False
  63
  64 def get(base, url, path, checksums, verbose=False, do_verify=True):
  65     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  66         temp_path = temp_file.name
  67
  68     try:
  69         if do_verify:
  70             if url not in checksums:
  71                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  72             sha256 = checksums[url]
  73             if os.path.exists(path):
  74                 if verify(path, sha256, False):
  75                     if verbose:
  76                         print("using already-download file", path)
  77                     return
  78                 else:
  79                     if verbose:
  80                         print("ignoring already-download file",
  81                             path, "due to failed verification")
  82                     os.unlink(path)
  83         download(temp_path, "{}/{}".format(base, url), True, verbose)
  84         if do_verify and not verify(temp_path, sha256, verbose):
  85             raise RuntimeError("failed verification")
  86         if verbose:
  87             print("moving {} to {}".format(temp_path, path))
  88         shutil.move(temp_path, path)
  89     finally:
  90         if os.path.isfile(temp_path):
  91             if verbose:
  92                 print("removing", temp_path)
  93             os.unlink(temp_path)
  94
  95
  96 def download(path, url, probably_big, verbose):
  97     for _ in range(0, 4):
  98         try:
  99             _download(path, url, probably_big, verbose, True)
 100             return
 101         except RuntimeError:
 102             print("\nspurious failure, trying again")
 103     _download(path, url, probably_big, verbose, False)
 104
 105
 106 def _download(path, url, probably_big, verbose, exception):
 107     if probably_big or verbose:
 108         print("downloading {}".format(url))
 109     # see https://serverfault.com/questions/301128/how-to-download
 110     if sys.platform == 'win32':
 111         run(["PowerShell.exe", "/nologo", "-Command",
 112              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 113              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
 114             verbose=verbose,
 115             exception=exception)
 116     else:
 117         if probably_big or verbose:
 118             option = "-#"
 119         else:
 120             option = "-s"
 121         require(["curl", "--version"])
 122         run(["curl", option,
 123              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
 124              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
 125              "--retry", "3", "-Sf", "-o", path, url],
 126             verbose=verbose,
 127             exception=exception)
 128
 129
 130 def verify(path, expected, verbose):
 131     """Check if the sha256 sum of the given path is valid"""
 132     if verbose:
 133         print("verifying", path)
 134     with open(path, "rb") as source:
 135         found = hashlib.sha256(source.read()).hexdigest()
 136     verified = found == expected
 137     if not verified:
 138         print("invalid checksum:\n"
 139               "    found:    {}\n"
 140               "    expected: {}".format(found, expected))
 141     return verified
 142
 143
 144 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 145     """Unpack the given tarball file"""
 146     print("extracting", tarball)
 147     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 148     with contextlib.closing(tarfile.open(tarball)) as tar:
 149         for member in tar.getnames():
 150             if "/" not in member:
 151                 continue
 152             name = member.replace(fname + "/", "", 1)
 153             if match is not None and not name.startswith(match):
 154                 continue
 155             name = name[len(match) + 1:]
 156
 157             dst_path = os.path.join(dst, name)
 158             if verbose:
 159                 print("  extracting", member)
 160             tar.extract(member, dst)
 161             src_path = os.path.join(dst, member)
 162             if os.path.isdir(src_path) and os.path.exists(dst_path):
 163                 continue
 164             shutil.move(src_path, dst_path)
 165     shutil.rmtree(os.path.join(dst, fname))
 166
 167
 168 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
 169     """Run a child program in a new process"""
 170     if verbose:
 171         print("running: " + ' '.join(args))
 172     sys.stdout.flush()
 173     # Use Popen here instead of call() as it apparently allows powershell on
 174     # Windows to not lock up waiting for input presumably.
 175     ret = subprocess.Popen(args, **kwargs)
 176     code = ret.wait()
 177     if code != 0:
 178         err = "failed to run: " + ' '.join(args)
 179         if verbose or exception:
 180             raise RuntimeError(err)
 181         # For most failures, we definitely do want to print this error, or the user will have no
 182         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 183         # have already printed an error above, so there's no need to print the exact command we're
 184         # running.
 185         if is_bootstrap:
 186             sys.exit(1)
 187         else:
 188             sys.exit(err)
 189
 190
 191 def require(cmd, exit=True):
 192     '''Run a command, returning its output.
 193     On error,
 194         If `exit` is `True`, exit the process.
 195         Otherwise, return None.'''
 196     try:
 197         return subprocess.check_output(cmd).strip()
 198     except (subprocess.CalledProcessError, OSError) as exc:
 199         if not exit:
 200             return None
 201         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 202         print("Please make sure it's installed and in the path.")
 203         sys.exit(1)
 204
 205
 206 def format_build_time(duration):
 207     """Return a nicer format for build time
 208
 209     >>> format_build_time('300')
 210     '0:05:00'
 211     """
 212     return str(datetime.timedelta(seconds=int(duration)))
 213
 214
 215 def default_build_triple(verbose):
 216     """Build triple as in LLVM"""
 217     # If the user already has a host build triple with an existing `rustc`
 218     # install, use their preference. This fixes most issues with Windows builds
 219     # being detected as GNU instead of MSVC.
 220     default_encoding = sys.getdefaultencoding()
 221     try:
 222         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 223                 stderr=subprocess.DEVNULL)
 224         version = version.decode(default_encoding)
 225         host = next(x for x in version.split('\n') if x.startswith("host: "))
 226         triple = host.split("host: ")[1]
 227         if verbose:
 228             print("detected default triple {} from pre-installed rustc".format(triple))
 229         return triple
 230     except Exception as e:
 231         if verbose:
 232             print("pre-installed rustc not detected: {}".format(e))
 233             print("falling back to auto-detect")
 234
 235     required = sys.platform != 'win32'
 236     ostype = require(["uname", "-s"], exit=required)
 237     cputype = require(['uname', '-m'], exit=required)
 238
 239     # If we do not have `uname`, assume Windows.
 240     if ostype is None or cputype is None:
 241         return 'x86_64-pc-windows-msvc'
 242
 243     ostype = ostype.decode(default_encoding)
 244     cputype = cputype.decode(default_encoding)
 245
 246     # The goal here is to come up with the same triple as LLVM would,
 247     # at least for the subset of platforms we're willing to target.
 248     ostype_mapper = {
 249         'Darwin': 'apple-darwin',
 250         'DragonFly': 'unknown-dragonfly',
 251         'FreeBSD': 'unknown-freebsd',
 252         'Haiku': 'unknown-haiku',
 253         'NetBSD': 'unknown-netbsd',
 254         'OpenBSD': 'unknown-openbsd'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if ostype in ostype_mapper:
 259         ostype = ostype_mapper[ostype]
 260     elif ostype == 'Linux':
 261         os_from_sp = subprocess.check_output(
 262             ['uname', '-o']).strip().decode(default_encoding)
 263         if os_from_sp == 'Android':
 264             ostype = 'linux-android'
 265         else:
 266             ostype = 'unknown-linux-gnu'
 267     elif ostype == 'SunOS':
 268         ostype = 'pc-solaris'
 269         # On Solaris, uname -m will return a machine classification instead
 270         # of a cpu type, so uname -p is recommended instead.  However, the
 271         # output from that option is too generic for our purposes (it will
 272         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 273         # must be used instead.
 274         cputype = require(['isainfo', '-k']).decode(default_encoding)
 275         # sparc cpus have sun as a target vendor
 276         if 'sparc' in cputype:
 277             ostype = 'sun-solaris'
 278     elif ostype.startswith('MINGW'):
 279         # msys' `uname` does not print gcc configuration, but prints msys
 280         # configuration. so we cannot believe `uname -m`:
 281         # msys1 is always i686 and msys2 is always x86_64.
 282         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 283         # MINGW64 on x86_64.
 284         ostype = 'pc-windows-gnu'
 285         cputype = 'i686'
 286         if os.environ.get('MSYSTEM') == 'MINGW64':
 287             cputype = 'x86_64'
 288     elif ostype.startswith('MSYS'):
 289         ostype = 'pc-windows-gnu'
 290     elif ostype.startswith('CYGWIN_NT'):
 291         cputype = 'i686'
 292         if ostype.endswith('WOW64'):
 293             cputype = 'x86_64'
 294         ostype = 'pc-windows-gnu'
 295     elif sys.platform == 'win32':
 296         # Some Windows platforms might have a `uname` command that returns a
 297         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 298         # these cases, fall back to using sys.platform.
 299         return 'x86_64-pc-windows-msvc'
 300     else:
 301         err = "unknown OS type: {}".format(ostype)
 302         sys.exit(err)
 303
 304     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
 305         cputype = subprocess.check_output(
 306               ['uname', '-p']).strip().decode(default_encoding)
 307     cputype_mapper = {
 308         'BePC': 'i686',
 309         'aarch64': 'aarch64',
 310         'amd64': 'x86_64',
 311         'arm64': 'aarch64',
 312         'i386': 'i686',
 313         'i486': 'i686',
 314         'i686': 'i686',
 315         'i786': 'i686',
 316         'm68k': 'm68k',
 317         'powerpc': 'powerpc',
 318         'powerpc64': 'powerpc64',
 319         'powerpc64le': 'powerpc64le',
 320         'ppc': 'powerpc',
 321         'ppc64': 'powerpc64',
 322         'ppc64le': 'powerpc64le',
 323         'riscv64': 'riscv64gc',
 324         's390x': 's390x',
 325         'x64': 'x86_64',
 326         'x86': 'i686',
 327         'x86-64': 'x86_64',
 328         'x86_64': 'x86_64'
 329     }
 330
 331     # Consider the direct transformation first and then the special cases
 332     if cputype in cputype_mapper:
 333         cputype = cputype_mapper[cputype]
 334     elif cputype in {'xscale', 'arm'}:
 335         cputype = 'arm'
 336         if ostype == 'linux-android':
 337             ostype = 'linux-androideabi'
 338         elif ostype == 'unknown-freebsd':
 339             cputype = subprocess.check_output(
 340                 ['uname', '-p']).strip().decode(default_encoding)
 341             ostype = 'unknown-freebsd'
 342     elif cputype == 'armv6l':
 343         cputype = 'arm'
 344         if ostype == 'linux-android':
 345             ostype = 'linux-androideabi'
 346         else:
 347             ostype += 'eabihf'
 348     elif cputype in {'armv7l', 'armv8l'}:
 349         cputype = 'armv7'
 350         if ostype == 'linux-android':
 351             ostype = 'linux-androideabi'
 352         else:
 353             ostype += 'eabihf'
 354     elif cputype == 'mips':
 355         if sys.byteorder == 'big':
 356             cputype = 'mips'
 357         elif sys.byteorder == 'little':
 358             cputype = 'mipsel'
 359         else:
 360             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 361     elif cputype == 'mips64':
 362         if sys.byteorder == 'big':
 363             cputype = 'mips64'
 364         elif sys.byteorder == 'little':
 365             cputype = 'mips64el'
 366         else:
 367             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 368         # only the n64 ABI is supported, indicate it
 369         ostype += 'abi64'
 370     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 371         pass
 372     else:
 373         err = "unknown cpu type: {}".format(cputype)
 374         sys.exit(err)
 375
 376     return "{}-{}".format(cputype, ostype)
 377
 378
 379 @contextlib.contextmanager
 380 def output(filepath):
 381     tmp = filepath + '.tmp'
 382     with open(tmp, 'w') as f:
 383         yield f
 384     try:
 385         if os.path.exists(filepath):
 386             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 387     except OSError:
 388         shutil.copy2(tmp, filepath)
 389         os.remove(tmp)
 390         return
 391     os.rename(tmp, filepath)
 392
 393
 394 class Stage0Toolchain:
 395     def __init__(self, stage0_payload):
 396         self.date = stage0_payload["date"]
 397         self.version = stage0_payload["version"]
 398
 399     def channel(self):
 400         return self.version + "-" + self.date
 401
 402
 403 class RustBuild(object):
 404     """Provide all the methods required to build Rust"""
 405     def __init__(self):
 406         self.checksums_sha256 = {}
 407         self.stage0_compiler = None
 408         self.stage0_rustfmt = None
 409         self._download_url = ''
 410         self.build = ''
 411         self.build_dir = ''
 412         self.clean = False
 413         self.config_toml = ''
 414         self.rust_root = ''
 415         self.use_locked_deps = ''
 416         self.use_vendored_sources = ''
 417         self.verbose = False
 418         self.git_version = None
 419         self.nix_deps_dir = None
 420         self.rustc_commit = None
 421
 422     def download_toolchain(self, stage0=True, rustc_channel=None):
 423         """Fetch the build system for Rust, written in Rust
 424
 425         This method will build a cache directory, then it will fetch the
 426         tarball which has the stage0 compiler used to then bootstrap the Rust
 427         compiler itself.
 428
 429         Each downloaded tarball is extracted, after that, the script
 430         will move all the content to the right place.
 431         """
 432         if rustc_channel is None:
 433             rustc_channel = self.stage0_compiler.version
 434         bin_root = self.bin_root(stage0)
 435
 436         key = self.stage0_compiler.date
 437         if not stage0:
 438             key += str(self.rustc_commit)
 439         if self.rustc(stage0).startswith(bin_root) and \
 440                 (not os.path.exists(self.rustc(stage0)) or
 441                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 442             if os.path.exists(bin_root):
 443                 shutil.rmtree(bin_root)
 444             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 445             filename = "rust-std-{}-{}{}".format(
 446                 rustc_channel, self.build, tarball_suffix)
 447             pattern = "rust-std-{}".format(self.build)
 448             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 449             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 450                                               tarball_suffix)
 451             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 452             # download-rustc doesn't need its own cargo, it can just use beta's.
 453             if stage0:
 454                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 455                                                 tarball_suffix)
 456                 self._download_component_helper(filename, "cargo", tarball_suffix)
 457                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 458             else:
 459                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 460                 self._download_component_helper(
 461                     filename, "rustc-dev", tarball_suffix, stage0
 462                 )
 463
 464             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 465             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 466             lib_dir = "{}/lib".format(bin_root)
 467             for lib in os.listdir(lib_dir):
 468                 if lib.endswith(".so"):
 469                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 470             with output(self.rustc_stamp(stage0)) as rust_stamp:
 471                 rust_stamp.write(key)
 472
 473         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 474             not os.path.exists(self.rustfmt())
 475             or self.program_out_of_date(
 476                 self.rustfmt_stamp(),
 477                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 478             )
 479         ):
 480             if self.stage0_rustfmt is not None:
 481                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 482                 filename = "rustfmt-{}-{}{}".format(
 483                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 484                 )
 485                 self._download_component_helper(
 486                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 487                 )
 488                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 489                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 490                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 491                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 492
 493         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 494         if self.downloading_llvm() and stage0:
 495             # We want the most recent LLVM submodule update to avoid downloading
 496             # LLVM more often than necessary.
 497             #
 498             # This git command finds that commit SHA, looking for bors-authored
 499             # commits that modified src/llvm-project or other relevant version
 500             # stamp files.
 501             #
 502             # This works even in a repository that has not yet initialized
 503             # submodules.
 504             top_level = subprocess.check_output([
 505                 "git", "rev-parse", "--show-toplevel",
 506             ]).decode(sys.getdefaultencoding()).strip()
 507             llvm_sha = subprocess.check_output([
 508                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 509                 "--first-parent", "HEAD",
 510                 "--",
 511                 "{}/src/llvm-project".format(top_level),
 512                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 513                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 514                 "{}/src/version".format(top_level)
 515             ]).decode(sys.getdefaultencoding()).strip()
 516             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 517             llvm_root = self.llvm_root()
 518             llvm_lib = os.path.join(llvm_root, "lib")
 519             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 520                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 521                 for binary in ["llvm-config", "FileCheck"]:
 522                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 523                 for lib in os.listdir(llvm_lib):
 524                     if lib.endswith(".so"):
 525                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 526                 with output(self.llvm_stamp()) as llvm_stamp:
 527                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 528
 529     def downloading_llvm(self):
 530         opt = self.get_toml('download-ci-llvm', 'llvm')
 531         # This is currently all tier 1 targets and tier 2 targets with host tools
 532         # (since others may not have CI artifacts)
 533         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 534         supported_platforms = [
 535             # tier 1
 536             "aarch64-unknown-linux-gnu",
 537             "i686-pc-windows-gnu",
 538             "i686-pc-windows-msvc",
 539             "i686-unknown-linux-gnu",
 540             "x86_64-unknown-linux-gnu",
 541             "x86_64-apple-darwin",
 542             "x86_64-pc-windows-gnu",
 543             "x86_64-pc-windows-msvc",
 544             # tier 2 with host tools
 545             "aarch64-apple-darwin",
 546             "aarch64-pc-windows-msvc",
 547             "aarch64-unknown-linux-musl",
 548             "arm-unknown-linux-gnueabi",
 549             "arm-unknown-linux-gnueabihf",
 550             "armv7-unknown-linux-gnueabihf",
 551             "mips-unknown-linux-gnu",
 552             "mips64-unknown-linux-gnuabi64",
 553             "mips64el-unknown-linux-gnuabi64",
 554             "mipsel-unknown-linux-gnu",
 555             "powerpc-unknown-linux-gnu",
 556             "powerpc64-unknown-linux-gnu",
 557             "powerpc64le-unknown-linux-gnu",
 558             "riscv64gc-unknown-linux-gnu",
 559             "s390x-unknown-linux-gnu",
 560             "x86_64-unknown-freebsd",
 561             "x86_64-unknown-illumos",
 562             "x86_64-unknown-linux-musl",
 563             "x86_64-unknown-netbsd",
 564         ]
 565         return opt == "true" \
 566             or (opt == "if-available" and self.build in supported_platforms)
 567
 568     def _download_component_helper(
 569         self, filename, pattern, tarball_suffix, stage0=True, key=None
 570     ):
 571         if key is None:
 572             if stage0:
 573                 key = self.stage0_compiler.date
 574             else:
 575                 key = self.rustc_commit
 576         cache_dst = os.path.join(self.build_dir, "cache")
 577         rustc_cache = os.path.join(cache_dst, key)
 578         if not os.path.exists(rustc_cache):
 579             os.makedirs(rustc_cache)
 580
 581         if stage0:
 582             base = self._download_url
 583             url = "dist/{}".format(key)
 584         else:
 585             base = "https://ci-artifacts.rust-lang.org"
 586             url = "rustc-builds/{}".format(self.rustc_commit)
 587         tarball = os.path.join(rustc_cache, filename)
 588         if not os.path.exists(tarball):
 589             get(
 590                 base,
 591                 "{}/{}".format(url, filename),
 592                 tarball,
 593                 self.checksums_sha256,
 594                 verbose=self.verbose,
 595                 do_verify=stage0,
 596             )
 597         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 598
 599     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 600         if not llvm_sha:
 601             print("error: could not find commit hash for downloading LLVM")
 602             print("help: maybe your repository history is too shallow?")
 603             print("help: consider disabling `download-ci-llvm`")
 604             print("help: or fetch enough history to include one upstream commit")
 605             exit(1)
 606         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 607         cache_dst = os.path.join(self.build_dir, "cache")
 608         rustc_cache = os.path.join(cache_dst, cache_prefix)
 609         if not os.path.exists(rustc_cache):
 610             os.makedirs(rustc_cache)
 611
 612         base = "https://ci-artifacts.rust-lang.org"
 613         url = "rustc-builds/{}".format(llvm_sha)
 614         if llvm_assertions:
 615             url = url.replace('rustc-builds', 'rustc-builds-alt')
 616         # ci-artifacts are only stored as .xz, not .gz
 617         if not support_xz():
 618             print("error: XZ support is required to download LLVM")
 619             print("help: consider disabling `download-ci-llvm` or using python3")
 620             exit(1)
 621         tarball_suffix = '.tar.xz'
 622         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 623         tarball = os.path.join(rustc_cache, filename)
 624         if not os.path.exists(tarball):
 625             get(
 626                 base,
 627                 "{}/{}".format(url, filename),
 628                 tarball,
 629                 self.checksums_sha256,
 630                 verbose=self.verbose,
 631                 do_verify=False,
 632             )
 633         unpack(tarball, tarball_suffix, self.llvm_root(),
 634                 match="rust-dev",
 635                 verbose=self.verbose)
 636
 637     def fix_bin_or_dylib(self, fname):
 638         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 639         or the RPATH section, to fix the dynamic library search path
 640
 641         This method is only required on NixOS and uses the PatchELF utility to
 642         change the interpreter/RPATH of ELF executables.
 643
 644         Please see https://nixos.org/patchelf.html for more information
 645         """
 646         default_encoding = sys.getdefaultencoding()
 647         try:
 648             ostype = subprocess.check_output(
 649                 ['uname', '-s']).strip().decode(default_encoding)
 650         except subprocess.CalledProcessError:
 651             return
 652         except OSError as reason:
 653             if getattr(reason, 'winerror', None) is not None:
 654                 return
 655             raise reason
 656
 657         if ostype != "Linux":
 658             return
 659
 660         # If the user has asked binaries to be patched for Nix, then
 661         # don't check for NixOS or `/lib`, just continue to the patching.
 662         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 663             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 664             # The latter one does not exist on NixOS when using tmpfs as root.
 665             try:
 666                 with open("/etc/os-release", "r") as f:
 667                     if not any(line.strip() == "ID=nixos" for line in f):
 668                         return
 669             except FileNotFoundError:
 670                 return
 671             if os.path.exists("/lib"):
 672                 return
 673
 674         # At this point we're pretty sure the user is running NixOS or
 675         # using Nix
 676         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 677         print(nix_os_msg, fname)
 678
 679         # Only build `.nix-deps` once.
 680         nix_deps_dir = self.nix_deps_dir
 681         if not nix_deps_dir:
 682             # Run `nix-build` to "build" each dependency (which will likely reuse
 683             # the existing `/nix/store` copy, or at most download a pre-built copy).
 684             #
 685             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 686             # directory, but still reference the actual `/nix/store` path in the rpath
 687             # as it makes it significantly more robust against changes to the location of
 688             # the `.nix-deps` location.
 689             #
 690             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 691             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 692             # patchelf: Needed for patching ELF binaries (see doc comment above).
 693             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 694             nix_expr = '''
 695             with (import <nixpkgs> {});
 696             symlinkJoin {
 697               name = "rust-stage0-dependencies";
 698               paths = [
 699                 zlib
 700                 patchelf
 701                 stdenv.cc.bintools
 702               ];
 703             }
 704             '''
 705             try:
 706                 subprocess.check_output([
 707                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 708                 ])
 709             except subprocess.CalledProcessError as reason:
 710                 print("warning: failed to call nix-build:", reason)
 711                 return
 712             self.nix_deps_dir = nix_deps_dir
 713
 714         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 715         rpath_entries = [
 716             # Relative default, all binary and dynamic libraries we ship
 717             # appear to have this (even when `../lib` is redundant).
 718             "$ORIGIN/../lib",
 719             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 720         ]
 721         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 722         if not fname.endswith(".so"):
 723             # Finally, set the corret .interp for binaries
 724             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 725                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 726
 727         try:
 728             subprocess.check_output([patchelf] + patchelf_args + [fname])
 729         except subprocess.CalledProcessError as reason:
 730             print("warning: failed to call patchelf:", reason)
 731             return
 732
 733     # If `download-rustc` is set, download the most recent commit with CI artifacts
 734     def maybe_download_ci_toolchain(self):
 735         # If `download-rustc` is not set, default to rebuilding.
 736         download_rustc = self.get_toml("download-rustc", section="rust")
 737         if download_rustc is None or download_rustc == "false":
 738             return None
 739         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 740
 741         # Handle running from a directory other than the top level
 742         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 743         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 744         compiler = "{}/compiler/".format(top_level)
 745         library = "{}/library/".format(top_level)
 746
 747         # Look for a version to compare to based on the current commit.
 748         # Only commits merged by bors will have CI artifacts.
 749         merge_base = [
 750             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 751             "--first-parent", "HEAD"
 752         ]
 753         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 754         if not commit:
 755             print("error: could not find commit hash for downloading rustc")
 756             print("help: maybe your repository history is too shallow?")
 757             print("help: consider disabling `download-rustc`")
 758             print("help: or fetch enough history to include one upstream commit")
 759             exit(1)
 760
 761         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 762         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 763         if status != 0:
 764             if download_rustc == "if-unchanged":
 765                 if self.verbose:
 766                     print("warning: saw changes to compiler/ or library/ since {}; " \
 767                           "ignoring `download-rustc`".format(commit))
 768                 return None
 769             print("warning: `download-rustc` is enabled, but there are changes to " \
 770                   "compiler/ or library/")
 771
 772         if self.verbose:
 773             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
 774         self.rustc_commit = commit
 775         # FIXME: support downloading artifacts from the beta channel
 776         self.download_toolchain(False, "nightly")
 777
 778     def rustc_stamp(self, stage0):
 779         """Return the path for .rustc-stamp at the given stage
 780
 781         >>> rb = RustBuild()
 782         >>> rb.build_dir = "build"
 783         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 784         True
 785         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 786         True
 787         """
 788         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 789
 790     def rustfmt_stamp(self):
 791         """Return the path for .rustfmt-stamp
 792
 793         >>> rb = RustBuild()
 794         >>> rb.build_dir = "build"
 795         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 796         True
 797         """
 798         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 799
 800     def llvm_stamp(self):
 801         """Return the path for .rustfmt-stamp
 802
 803         >>> rb = RustBuild()
 804         >>> rb.build_dir = "build"
 805         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 806         True
 807         """
 808         return os.path.join(self.llvm_root(), '.llvm-stamp')
 809
 810
 811     def program_out_of_date(self, stamp_path, key):
 812         """Check if the given program stamp is out of date"""
 813         if not os.path.exists(stamp_path) or self.clean:
 814             return True
 815         with open(stamp_path, 'r') as stamp:
 816             return key != stamp.read()
 817
 818     def bin_root(self, stage0):
 819         """Return the binary root directory for the given stage
 820
 821         >>> rb = RustBuild()
 822         >>> rb.build_dir = "build"
 823         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 824         True
 825         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 826         True
 827
 828         When the 'build' property is given should be a nested directory:
 829
 830         >>> rb.build = "devel"
 831         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 832         True
 833         """
 834         if stage0:
 835             subdir = "stage0"
 836         else:
 837             subdir = "ci-rustc"
 838         return os.path.join(self.build_dir, self.build, subdir)
 839
 840     def llvm_root(self):
 841         """Return the CI LLVM root directory
 842
 843         >>> rb = RustBuild()
 844         >>> rb.build_dir = "build"
 845         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 846         True
 847
 848         When the 'build' property is given should be a nested directory:
 849
 850         >>> rb.build = "devel"
 851         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 852         True
 853         """
 854         return os.path.join(self.build_dir, self.build, "ci-llvm")
 855
 856     def get_toml(self, key, section=None):
 857         """Returns the value of the given key in config.toml, otherwise returns None
 858
 859         >>> rb = RustBuild()
 860         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 861         >>> rb.get_toml("key2")
 862         'value2'
 863
 864         If the key does not exists, the result is None:
 865
 866         >>> rb.get_toml("key3") is None
 867         True
 868
 869         Optionally also matches the section the key appears in
 870
 871         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 872         >>> rb.get_toml('key', 'a')
 873         'value1'
 874         >>> rb.get_toml('key', 'b')
 875         'value2'
 876         >>> rb.get_toml('key', 'c') is None
 877         True
 878
 879         >>> rb.config_toml = 'key1 = true'
 880         >>> rb.get_toml("key1")
 881         'true'
 882         """
 883
 884         cur_section = None
 885         for line in self.config_toml.splitlines():
 886             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 887             if section_match is not None:
 888                 cur_section = section_match.group(1)
 889
 890             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 891             if match is not None:
 892                 value = match.group(1)
 893                 if section is None or section == cur_section:
 894                     return self.get_string(value) or value.strip()
 895         return None
 896
 897     def cargo(self):
 898         """Return config path for cargo"""
 899         return self.program_config('cargo')
 900
 901     def rustc(self, stage0):
 902         """Return config path for rustc"""
 903         return self.program_config('rustc', stage0)
 904
 905     def rustfmt(self):
 906         """Return config path for rustfmt"""
 907         if self.stage0_rustfmt is None:
 908             return None
 909         return self.program_config('rustfmt')
 910
 911     def program_config(self, program, stage0=True):
 912         """Return config path for the given program at the given stage
 913
 914         >>> rb = RustBuild()
 915         >>> rb.config_toml = 'rustc = "rustc"\\n'
 916         >>> rb.program_config('rustc')
 917         'rustc'
 918         >>> rb.config_toml = ''
 919         >>> cargo_path = rb.program_config('cargo', True)
 920         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 921         ... "bin", "cargo")
 922         True
 923         >>> cargo_path = rb.program_config('cargo', False)
 924         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 925         ... "bin", "cargo")
 926         True
 927         """
 928         config = self.get_toml(program)
 929         if config:
 930             return os.path.expanduser(config)
 931         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 932             program, self.exe_suffix()))
 933
 934     @staticmethod
 935     def get_string(line):
 936         """Return the value between double quotes
 937
 938         >>> RustBuild.get_string('    "devel"   ')
 939         'devel'
 940         >>> RustBuild.get_string("    'devel'   ")
 941         'devel'
 942         >>> RustBuild.get_string('devel') is None
 943         True
 944         >>> RustBuild.get_string('    "devel   ')
 945         ''
 946         """
 947         start = line.find('"')
 948         if start != -1:
 949             end = start + 1 + line[start + 1:].find('"')
 950             return line[start + 1:end]
 951         start = line.find('\'')
 952         if start != -1:
 953             end = start + 1 + line[start + 1:].find('\'')
 954             return line[start + 1:end]
 955         return None
 956
 957     @staticmethod
 958     def exe_suffix():
 959         """Return a suffix for executables"""
 960         if sys.platform == 'win32':
 961             return '.exe'
 962         return ''
 963
 964     def bootstrap_binary(self):
 965         """Return the path of the bootstrap binary
 966
 967         >>> rb = RustBuild()
 968         >>> rb.build_dir = "build"
 969         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 970         ... "debug", "bootstrap")
 971         True
 972         """
 973         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 974
 975     def build_bootstrap(self):
 976         """Build bootstrap"""
 977         print("Building rustbuild")
 978         build_dir = os.path.join(self.build_dir, "bootstrap")
 979         if self.clean and os.path.exists(build_dir):
 980             shutil.rmtree(build_dir)
 981         env = os.environ.copy()
 982         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 983         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 984         if "CARGO_BUILD_TARGET" in env:
 985             del env["CARGO_BUILD_TARGET"]
 986         env["CARGO_TARGET_DIR"] = build_dir
 987         env["RUSTC"] = self.rustc(True)
 988         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 989             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 990             if "LD_LIBRARY_PATH" in env else ""
 991         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 992             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 993             if "DYLD_LIBRARY_PATH" in env else ""
 994         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 995             (os.pathsep + env["LIBRARY_PATH"]) \
 996             if "LIBRARY_PATH" in env else ""
 997
 998         # preserve existing RUSTFLAGS
 999         env.setdefault("RUSTFLAGS", "")
1000         build_section = "target.{}".format(self.build)
1001         target_features = []
1002         if self.get_toml("crt-static", build_section) == "true":
1003             target_features += ["+crt-static"]
1004         elif self.get_toml("crt-static", build_section) == "false":
1005             target_features += ["-crt-static"]
1006         if target_features:
1007             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1008         target_linker = self.get_toml("linker", build_section)
1009         if target_linker is not None:
1010             env["RUSTFLAGS"] += " -C linker=" + target_linker
1011         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1012         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1013         if self.get_toml("deny-warnings", "rust") != "false":
1014             env["RUSTFLAGS"] += " -Dwarnings"
1015
1016         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1017             os.pathsep + env["PATH"]
1018         if not os.path.isfile(self.cargo()):
1019             raise Exception("no cargo executable found at `{}`".format(
1020                 self.cargo()))
1021         args = [self.cargo(), "build", "--manifest-path",
1022                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1023         for _ in range(0, self.verbose):
1024             args.append("--verbose")
1025         if self.use_locked_deps:
1026             args.append("--locked")
1027         if self.use_vendored_sources:
1028             args.append("--frozen")
1029         run(args, env=env, verbose=self.verbose)
1030
1031     def build_triple(self):
1032         """Build triple as in LLVM
1033
1034         Note that `default_build_triple` is moderately expensive,
1035         so use `self.build` where possible.
1036         """
1037         config = self.get_toml('build')
1038         if config:
1039             return config
1040         return default_build_triple(self.verbose)
1041
1042     def check_submodule(self, module, slow_submodules):
1043         if not slow_submodules:
1044             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1045                                            cwd=os.path.join(self.rust_root, module),
1046                                            stdout=subprocess.PIPE)
1047             return checked_out
1048         else:
1049             return None
1050
1051     def update_submodule(self, module, checked_out, recorded_submodules):
1052         module_path = os.path.join(self.rust_root, module)
1053
1054         if checked_out is not None:
1055             default_encoding = sys.getdefaultencoding()
1056             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1057             if recorded_submodules[module] == checked_out:
1058                 return
1059
1060         print("Updating submodule", module)
1061
1062         run(["git", "submodule", "-q", "sync", module],
1063             cwd=self.rust_root, verbose=self.verbose)
1064
1065         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1066         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1067             update_args.append("--progress")
1068         update_args.append(module)
1069         try:
1070             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1071         except RuntimeError:
1072             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1073             print('Either stash the changes by running "git stash" within the submodule\'s')
1074             print('directory, reset them by running "git reset --hard", or commit them.')
1075             print("To reset all submodules' changes run", end=" ")
1076             print('"git submodule foreach --recursive git reset --hard".')
1077             raise SystemExit(1)
1078
1079         run(["git", "reset", "-q", "--hard"],
1080             cwd=module_path, verbose=self.verbose)
1081         run(["git", "clean", "-qdfx"],
1082             cwd=module_path, verbose=self.verbose)
1083
1084     def update_submodules(self):
1085         """Update submodules"""
1086         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1087                 self.get_toml('submodules') == "false":
1088             return
1089
1090         default_encoding = sys.getdefaultencoding()
1091
1092         # check the existence and version of 'git' command
1093         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1094         self.git_version = distutils.version.LooseVersion(git_version_str)
1095
1096         slow_submodules = self.get_toml('fast-submodules') == "false"
1097         start_time = time()
1098         if slow_submodules:
1099             print('Unconditionally updating submodules')
1100         else:
1101             print('Updating only changed submodules')
1102         default_encoding = sys.getdefaultencoding()
1103         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1104         # currently requires everything in a workspace to be "locally present" when starting a
1105         # build, and will give a hard error if any Cargo.toml files are missing.
1106         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1107         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1108         #   That will still require cloning the submodules the second you check the standard
1109         #   library, though...
1110         # FIXME: Is there a way to avoid hard-coding the submodules required?
1111         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1112         submodules = [
1113             "src/tools/rust-installer",
1114             "src/tools/cargo",
1115             "src/tools/rls",
1116             "src/tools/miri",
1117             "library/backtrace",
1118             "library/stdarch"
1119         ]
1120         filtered_submodules = []
1121         submodules_names = []
1122         for module in submodules:
1123             check = self.check_submodule(module, slow_submodules)
1124             filtered_submodules.append((module, check))
1125             submodules_names.append(module)
1126         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1127                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1128         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1129         # { filename: hash }
1130         recorded_submodules = {}
1131         for data in recorded:
1132             # [mode, kind, hash, filename]
1133             data = data.split()
1134             recorded_submodules[data[3]] = data[2]
1135         for module in filtered_submodules:
1136             self.update_submodule(module[0], module[1], recorded_submodules)
1137         print("  Submodules updated in %.2f seconds" % (time() - start_time))
1138
1139     def set_dist_environment(self, url):
1140         """Set download URL for normal environment"""
1141         if 'RUSTUP_DIST_SERVER' in os.environ:
1142             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1143         else:
1144             self._download_url = url
1145
1146     def check_vendored_status(self):
1147         """Check that vendoring is configured properly"""
1148         vendor_dir = os.path.join(self.rust_root, 'vendor')
1149         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1150             if os.environ.get('USER') != os.environ['SUDO_USER']:
1151                 self.use_vendored_sources = True
1152                 print('info: looks like you are running this command under `sudo`')
1153                 print('      and so in order to preserve your $HOME this will now')
1154                 print('      use vendored sources by default.')
1155                 if not os.path.exists(vendor_dir):
1156                     print('error: vendoring required, but vendor directory does not exist.')
1157                     print('       Run `cargo vendor` without sudo to initialize the '
1158                           'vendor directory.')
1159                     raise Exception("{} not found".format(vendor_dir))
1160
1161         if self.use_vendored_sources:
1162             config = ("[source.crates-io]\n"
1163                       "replace-with = 'vendored-sources'\n"
1164                       "registry = 'https://example.com'\n"
1165                       "\n"
1166                       "[source.vendored-sources]\n"
1167                       "directory = '{}/vendor'\n"
1168                       .format(self.rust_root))
1169             if not os.path.exists('.cargo'):
1170                 os.makedirs('.cargo')
1171                 with output('.cargo/config') as cargo_config:
1172                     cargo_config.write(config)
1173             else:
1174                 print('info: using vendored source, but .cargo/config is already present.')
1175                 print('      Reusing the current configuration file. But you may want to '
1176                       'configure vendoring like this:')
1177                 print(config)
1178         else:
1179             if os.path.exists('.cargo'):
1180                 shutil.rmtree('.cargo')
1181
1182     def ensure_vendored(self):
1183         """Ensure that the vendored sources are available if needed"""
1184         vendor_dir = os.path.join(self.rust_root, 'vendor')
1185         # Note that this does not handle updating the vendored dependencies if
1186         # the rust git repository is updated. Normal development usually does
1187         # not use vendoring, so hopefully this isn't too much of a problem.
1188         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1189             run([
1190                 self.cargo(),
1191                 "vendor",
1192                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1193                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1194             ], verbose=self.verbose, cwd=self.rust_root)
1195
1196
1197 def bootstrap(help_triggered):
1198     """Configure, fetch, build and run the initial bootstrap"""
1199
1200     # If the user is asking for help, let them know that the whole download-and-build
1201     # process has to happen before anything is printed out.
1202     if help_triggered:
1203         print("info: Downloading and building bootstrap before processing --help")
1204         print("      command. See src/bootstrap/README.md for help with common")
1205         print("      commands.")
1206
1207     parser = argparse.ArgumentParser(description='Build rust')
1208     parser.add_argument('--config')
1209     parser.add_argument('--build')
1210     parser.add_argument('--clean', action='store_true')
1211     parser.add_argument('-v', '--verbose', action='count', default=0)
1212
1213     args = [a for a in sys.argv if a != '-h' and a != '--help']
1214     args, _ = parser.parse_known_args(args)
1215
1216     # Configure initial bootstrap
1217     build = RustBuild()
1218     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1219     build.verbose = args.verbose
1220     build.clean = args.clean
1221
1222     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1223     # exists).
1224     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1225     if not toml_path and os.path.exists('config.toml'):
1226         toml_path = 'config.toml'
1227
1228     if toml_path:
1229         if not os.path.exists(toml_path):
1230             toml_path = os.path.join(build.rust_root, toml_path)
1231
1232         with open(toml_path) as config:
1233             build.config_toml = config.read()
1234
1235     profile = build.get_toml('profile')
1236     if profile is not None:
1237         include_file = 'config.{}.toml'.format(profile)
1238         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1239         include_path = os.path.join(include_dir, include_file)
1240         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1241         # specific key, so appending our defaults at the end allows the user to override them
1242         with open(include_path) as included_toml:
1243             build.config_toml += os.linesep + included_toml.read()
1244
1245     config_verbose = build.get_toml('verbose', 'build')
1246     if config_verbose is not None:
1247         build.verbose = max(build.verbose, int(config_verbose))
1248
1249     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1250
1251     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1252
1253     build.check_vendored_status()
1254
1255     build_dir = build.get_toml('build-dir', 'build') or 'build'
1256     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1257
1258     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1259         data = json.load(f)
1260     build.checksums_sha256 = data["checksums_sha256"]
1261     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1262     if data.get("rustfmt") is not None:
1263         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1264
1265     build.set_dist_environment(data["dist_server"])
1266
1267     build.build = args.build or build.build_triple()
1268
1269     # Acquire the lock before doing any build actions
1270     # The lock is released when `lock` is dropped
1271     if not os.path.exists(build.build_dir):
1272         os.makedirs(build.build_dir)
1273     lock = acquire_lock(build.build_dir)
1274     build.update_submodules()
1275
1276     # Fetch/build the bootstrap
1277     build.download_toolchain()
1278     # Download the master compiler if `download-rustc` is set
1279     build.maybe_download_ci_toolchain()
1280     sys.stdout.flush()
1281     build.ensure_vendored()
1282     build.build_bootstrap()
1283     sys.stdout.flush()
1284
1285     # Run the bootstrap
1286     args = [build.bootstrap_binary()]
1287     args.extend(sys.argv[1:])
1288     env = os.environ.copy()
1289     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1290     env["BOOTSTRAP_PYTHON"] = sys.executable
1291     env["BUILD_DIR"] = build.build_dir
1292     env["RUSTC_BOOTSTRAP"] = '1'
1293     if toml_path:
1294         env["BOOTSTRAP_CONFIG"] = toml_path
1295     if build.rustc_commit is not None:
1296         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1297     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1298
1299
1300 def main():
1301     """Entry point for the bootstrap process"""
1302     start_time = time()
1303
1304     # x.py help <cmd> ...
1305     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1306         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1307
1308     help_triggered = (
1309         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1310     try:
1311         bootstrap(help_triggered)
1312         if not help_triggered:
1313             print("Build completed successfully in {}".format(
1314                 format_build_time(time() - start_time)))
1315     except (SystemExit, KeyboardInterrupt) as error:
1316         if hasattr(error, 'code') and isinstance(error.code, int):
1317             exit_code = error.code
1318         else:
1319             exit_code = 1
1320             print(error)
1321         if not help_triggered:
1322             print("Build completed unsuccessfully in {}".format(
1323                 format_build_time(time() - start_time)))
1324         sys.exit(exit_code)
1325
1326
1327 if __name__ == '__main__':
1328     main()