src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time, sleep
  17
  18 # Acquire a lock on the build directory to make sure that
  19 # we don't cause a race condition while building
  20 # Lock is created in `build_dir/lock.db`
  21 def acquire_lock(build_dir):
  22     try:
  23         import sqlite3
  24
  25         path = os.path.join(build_dir, "lock.db")
  26         try:
  27             con = sqlite3.Connection(path, timeout=0)
  28             curs = con.cursor()
  29             curs.execute("BEGIN EXCLUSIVE")
  30             # The lock is released when the cursor is dropped
  31             return curs
  32         # If the database is busy then lock has already been acquired
  33         # so we wait for the lock.
  34         # We retry every quarter second so that execution is passed back to python
  35         # so that it can handle signals
  36         except sqlite3.OperationalError:
  37             del con
  38             del curs
  39             print("Waiting for lock on build directory")
  40             con = sqlite3.Connection(path, timeout=0.25)
  41             curs = con.cursor()
  42             while True:
  43                 try:
  44                     curs.execute("BEGIN EXCLUSIVE")
  45                     break
  46                 except sqlite3.OperationalError:
  47                     pass
  48                 sleep(0.25)
  49             return curs
  50     except ImportError:
  51         print("warning: sqlite3 not available in python, skipping build directory lock")
  52         print("please file an issue on rust-lang/rust")
  53         print("this is not a problem for non-concurrent x.py invocations")
  54         return None
  55
  56 def support_xz():
  57     try:
  58         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  59             temp_path = temp_file.name
  60         with tarfile.open(temp_path, "w:xz"):
  61             pass
  62         return True
  63     except tarfile.CompressionError:
  64         return False
  65
  66 def get(base, url, path, checksums, verbose=False, do_verify=True, help_on_error=None):
  67     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  68         temp_path = temp_file.name
  69
  70     try:
  71         if do_verify:
  72             if url not in checksums:
  73                 raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
  74                                     "Pre-built artifacts might not available for this "
  75                                     "target at this time, see https://doc.rust-lang.org/nightly"
  76                                     "/rustc/platform-support.html for more information.")
  77                                    .format(url))
  78             sha256 = checksums[url]
  79             if os.path.exists(path):
  80                 if verify(path, sha256, False):
  81                     if verbose:
  82                         print("using already-download file", path)
  83                     return
  84                 else:
  85                     if verbose:
  86                         print("ignoring already-download file",
  87                             path, "due to failed verification")
  88                     os.unlink(path)
  89         download(temp_path, "{}/{}".format(base, url), True, verbose, help_on_error=help_on_error)
  90         if do_verify and not verify(temp_path, sha256, verbose):
  91             raise RuntimeError("failed verification")
  92         if verbose:
  93             print("moving {} to {}".format(temp_path, path))
  94         shutil.move(temp_path, path)
  95     finally:
  96         if os.path.isfile(temp_path):
  97             if verbose:
  98                 print("removing", temp_path)
  99             os.unlink(temp_path)
 100
 101
 102 def download(path, url, probably_big, verbose, help_on_error=None):
 103     for _ in range(0, 4):
 104         try:
 105             _download(path, url, probably_big, verbose, True, help_on_error=help_on_error)
 106             return
 107         except RuntimeError:
 108             print("\nspurious failure, trying again")
 109     _download(path, url, probably_big, verbose, False, help_on_error=help_on_error)
 110
 111
 112 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
 113     if probably_big or verbose:
 114         print("downloading {}".format(url))
 115     # see https://serverfault.com/questions/301128/how-to-download
 116     if sys.platform == 'win32':
 117         run(["PowerShell.exe", "/nologo", "-Command",
 118              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 119              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
 120             verbose=verbose,
 121             exception=exception)
 122     else:
 123         if probably_big or verbose:
 124             option = "-#"
 125         else:
 126             option = "-s"
 127         require(["curl", "--version"])
 128         run(["curl", option,
 129              "-L", # Follow redirect.
 130              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
 131              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
 132              "--retry", "3", "-Sf", "-o", path, url],
 133             verbose=verbose,
 134             exception=exception,
 135             help_on_error=help_on_error)
 136
 137
 138 def verify(path, expected, verbose):
 139     """Check if the sha256 sum of the given path is valid"""
 140     if verbose:
 141         print("verifying", path)
 142     with open(path, "rb") as source:
 143         found = hashlib.sha256(source.read()).hexdigest()
 144     verified = found == expected
 145     if not verified:
 146         print("invalid checksum:\n"
 147               "    found:    {}\n"
 148               "    expected: {}".format(found, expected))
 149     return verified
 150
 151
 152 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 153     """Unpack the given tarball file"""
 154     print("extracting", tarball)
 155     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 156     with contextlib.closing(tarfile.open(tarball)) as tar:
 157         for member in tar.getnames():
 158             if "/" not in member:
 159                 continue
 160             name = member.replace(fname + "/", "", 1)
 161             if match is not None and not name.startswith(match):
 162                 continue
 163             name = name[len(match) + 1:]
 164
 165             dst_path = os.path.join(dst, name)
 166             if verbose:
 167                 print("  extracting", member)
 168             tar.extract(member, dst)
 169             src_path = os.path.join(dst, member)
 170             if os.path.isdir(src_path) and os.path.exists(dst_path):
 171                 continue
 172             shutil.move(src_path, dst_path)
 173     shutil.rmtree(os.path.join(dst, fname))
 174
 175
 176 def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=None, **kwargs):
 177     """Run a child program in a new process"""
 178     if verbose:
 179         print("running: " + ' '.join(args))
 180     sys.stdout.flush()
 181     # Use Popen here instead of call() as it apparently allows powershell on
 182     # Windows to not lock up waiting for input presumably.
 183     ret = subprocess.Popen(args, **kwargs)
 184     code = ret.wait()
 185     if code != 0:
 186         err = "failed to run: " + ' '.join(args)
 187         if help_on_error is not None:
 188             err += "\n" + help_on_error
 189         if verbose or exception:
 190             raise RuntimeError(err)
 191         # For most failures, we definitely do want to print this error, or the user will have no
 192         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 193         # have already printed an error above, so there's no need to print the exact command we're
 194         # running.
 195         if is_bootstrap:
 196             sys.exit(1)
 197         else:
 198             sys.exit(err)
 199
 200
 201 def require(cmd, exit=True):
 202     '''Run a command, returning its output.
 203     On error,
 204         If `exit` is `True`, exit the process.
 205         Otherwise, return None.'''
 206     try:
 207         return subprocess.check_output(cmd).strip()
 208     except (subprocess.CalledProcessError, OSError) as exc:
 209         if not exit:
 210             return None
 211         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 212         print("Please make sure it's installed and in the path.")
 213         sys.exit(1)
 214
 215
 216 def format_build_time(duration):
 217     """Return a nicer format for build time
 218
 219     >>> format_build_time('300')
 220     '0:05:00'
 221     """
 222     return str(datetime.timedelta(seconds=int(duration)))
 223
 224
 225 def default_build_triple(verbose):
 226     """Build triple as in LLVM"""
 227     # If the user already has a host build triple with an existing `rustc`
 228     # install, use their preference. This fixes most issues with Windows builds
 229     # being detected as GNU instead of MSVC.
 230     default_encoding = sys.getdefaultencoding()
 231     try:
 232         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 233                 stderr=subprocess.DEVNULL)
 234         version = version.decode(default_encoding)
 235         host = next(x for x in version.split('\n') if x.startswith("host: "))
 236         triple = host.split("host: ")[1]
 237         if verbose:
 238             print("detected default triple {} from pre-installed rustc".format(triple))
 239         return triple
 240     except Exception as e:
 241         if verbose:
 242             print("pre-installed rustc not detected: {}".format(e))
 243             print("falling back to auto-detect")
 244
 245     required = sys.platform != 'win32'
 246     ostype = require(["uname", "-s"], exit=required)
 247     cputype = require(['uname', '-m'], exit=required)
 248
 249     # If we do not have `uname`, assume Windows.
 250     if ostype is None or cputype is None:
 251         return 'x86_64-pc-windows-msvc'
 252
 253     ostype = ostype.decode(default_encoding)
 254     cputype = cputype.decode(default_encoding)
 255
 256     # The goal here is to come up with the same triple as LLVM would,
 257     # at least for the subset of platforms we're willing to target.
 258     ostype_mapper = {
 259         'Darwin': 'apple-darwin',
 260         'DragonFly': 'unknown-dragonfly',
 261         'FreeBSD': 'unknown-freebsd',
 262         'Haiku': 'unknown-haiku',
 263         'NetBSD': 'unknown-netbsd',
 264         'OpenBSD': 'unknown-openbsd'
 265     }
 266
 267     # Consider the direct transformation first and then the special cases
 268     if ostype in ostype_mapper:
 269         ostype = ostype_mapper[ostype]
 270     elif ostype == 'Linux':
 271         os_from_sp = subprocess.check_output(
 272             ['uname', '-o']).strip().decode(default_encoding)
 273         if os_from_sp == 'Android':
 274             ostype = 'linux-android'
 275         else:
 276             ostype = 'unknown-linux-gnu'
 277     elif ostype == 'SunOS':
 278         ostype = 'pc-solaris'
 279         # On Solaris, uname -m will return a machine classification instead
 280         # of a cpu type, so uname -p is recommended instead.  However, the
 281         # output from that option is too generic for our purposes (it will
 282         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 283         # must be used instead.
 284         cputype = require(['isainfo', '-k']).decode(default_encoding)
 285         # sparc cpus have sun as a target vendor
 286         if 'sparc' in cputype:
 287             ostype = 'sun-solaris'
 288     elif ostype.startswith('MINGW'):
 289         # msys' `uname` does not print gcc configuration, but prints msys
 290         # configuration. so we cannot believe `uname -m`:
 291         # msys1 is always i686 and msys2 is always x86_64.
 292         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 293         # MINGW64 on x86_64.
 294         ostype = 'pc-windows-gnu'
 295         cputype = 'i686'
 296         if os.environ.get('MSYSTEM') == 'MINGW64':
 297             cputype = 'x86_64'
 298     elif ostype.startswith('MSYS'):
 299         ostype = 'pc-windows-gnu'
 300     elif ostype.startswith('CYGWIN_NT'):
 301         cputype = 'i686'
 302         if ostype.endswith('WOW64'):
 303             cputype = 'x86_64'
 304         ostype = 'pc-windows-gnu'
 305     elif sys.platform == 'win32':
 306         # Some Windows platforms might have a `uname` command that returns a
 307         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 308         # these cases, fall back to using sys.platform.
 309         return 'x86_64-pc-windows-msvc'
 310     else:
 311         err = "unknown OS type: {}".format(ostype)
 312         sys.exit(err)
 313
 314     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
 315         cputype = subprocess.check_output(
 316               ['uname', '-p']).strip().decode(default_encoding)
 317     cputype_mapper = {
 318         'BePC': 'i686',
 319         'aarch64': 'aarch64',
 320         'amd64': 'x86_64',
 321         'arm64': 'aarch64',
 322         'i386': 'i686',
 323         'i486': 'i686',
 324         'i686': 'i686',
 325         'i786': 'i686',
 326         'm68k': 'm68k',
 327         'powerpc': 'powerpc',
 328         'powerpc64': 'powerpc64',
 329         'powerpc64le': 'powerpc64le',
 330         'ppc': 'powerpc',
 331         'ppc64': 'powerpc64',
 332         'ppc64le': 'powerpc64le',
 333         'riscv64': 'riscv64gc',
 334         's390x': 's390x',
 335         'x64': 'x86_64',
 336         'x86': 'i686',
 337         'x86-64': 'x86_64',
 338         'x86_64': 'x86_64'
 339     }
 340
 341     # Consider the direct transformation first and then the special cases
 342     if cputype in cputype_mapper:
 343         cputype = cputype_mapper[cputype]
 344     elif cputype in {'xscale', 'arm'}:
 345         cputype = 'arm'
 346         if ostype == 'linux-android':
 347             ostype = 'linux-androideabi'
 348         elif ostype == 'unknown-freebsd':
 349             cputype = subprocess.check_output(
 350                 ['uname', '-p']).strip().decode(default_encoding)
 351             ostype = 'unknown-freebsd'
 352     elif cputype == 'armv6l':
 353         cputype = 'arm'
 354         if ostype == 'linux-android':
 355             ostype = 'linux-androideabi'
 356         else:
 357             ostype += 'eabihf'
 358     elif cputype in {'armv7l', 'armv8l'}:
 359         cputype = 'armv7'
 360         if ostype == 'linux-android':
 361             ostype = 'linux-androideabi'
 362         else:
 363             ostype += 'eabihf'
 364     elif cputype == 'mips':
 365         if sys.byteorder == 'big':
 366             cputype = 'mips'
 367         elif sys.byteorder == 'little':
 368             cputype = 'mipsel'
 369         else:
 370             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 371     elif cputype == 'mips64':
 372         if sys.byteorder == 'big':
 373             cputype = 'mips64'
 374         elif sys.byteorder == 'little':
 375             cputype = 'mips64el'
 376         else:
 377             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 378         # only the n64 ABI is supported, indicate it
 379         ostype += 'abi64'
 380     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 381         pass
 382     else:
 383         err = "unknown cpu type: {}".format(cputype)
 384         sys.exit(err)
 385
 386     return "{}-{}".format(cputype, ostype)
 387
 388
 389 @contextlib.contextmanager
 390 def output(filepath):
 391     tmp = filepath + '.tmp'
 392     with open(tmp, 'w') as f:
 393         yield f
 394     try:
 395         if os.path.exists(filepath):
 396             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 397     except OSError:
 398         shutil.copy2(tmp, filepath)
 399         os.remove(tmp)
 400         return
 401     os.rename(tmp, filepath)
 402
 403
 404 class Stage0Toolchain:
 405     def __init__(self, stage0_payload):
 406         self.date = stage0_payload["date"]
 407         self.version = stage0_payload["version"]
 408
 409     def channel(self):
 410         return self.version + "-" + self.date
 411
 412
 413 class RustBuild(object):
 414     """Provide all the methods required to build Rust"""
 415     def __init__(self):
 416         self.checksums_sha256 = {}
 417         self.stage0_compiler = None
 418         self.stage0_rustfmt = None
 419         self._download_url = ''
 420         self.build = ''
 421         self.build_dir = ''
 422         self.clean = False
 423         self.config_toml = ''
 424         self.rust_root = ''
 425         self.use_locked_deps = ''
 426         self.use_vendored_sources = ''
 427         self.verbose = False
 428         self.git_version = None
 429         self.nix_deps_dir = None
 430         self.rustc_commit = None
 431
 432     def download_toolchain(self, stage0=True, rustc_channel=None):
 433         """Fetch the build system for Rust, written in Rust
 434
 435         This method will build a cache directory, then it will fetch the
 436         tarball which has the stage0 compiler used to then bootstrap the Rust
 437         compiler itself.
 438
 439         Each downloaded tarball is extracted, after that, the script
 440         will move all the content to the right place.
 441         """
 442         if rustc_channel is None:
 443             rustc_channel = self.stage0_compiler.version
 444         bin_root = self.bin_root(stage0)
 445
 446         key = self.stage0_compiler.date
 447         if not stage0:
 448             key += str(self.rustc_commit)
 449         if self.rustc(stage0).startswith(bin_root) and \
 450                 (not os.path.exists(self.rustc(stage0)) or
 451                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 452             if os.path.exists(bin_root):
 453                 shutil.rmtree(bin_root)
 454             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 455             filename = "rust-std-{}-{}{}".format(
 456                 rustc_channel, self.build, tarball_suffix)
 457             pattern = "rust-std-{}".format(self.build)
 458             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 459             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 460                                               tarball_suffix)
 461             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 462             # download-rustc doesn't need its own cargo, it can just use beta's.
 463             if stage0:
 464                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 465                                                 tarball_suffix)
 466                 self._download_component_helper(filename, "cargo", tarball_suffix)
 467                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 468             else:
 469                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 470                 self._download_component_helper(
 471                     filename, "rustc-dev", tarball_suffix, stage0
 472                 )
 473
 474             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 475             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 476             lib_dir = "{}/lib".format(bin_root)
 477             for lib in os.listdir(lib_dir):
 478                 if lib.endswith(".so"):
 479                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 480             with output(self.rustc_stamp(stage0)) as rust_stamp:
 481                 rust_stamp.write(key)
 482
 483         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 484             not os.path.exists(self.rustfmt())
 485             or self.program_out_of_date(
 486                 self.rustfmt_stamp(),
 487                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 488             )
 489         ):
 490             if self.stage0_rustfmt is not None:
 491                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 492                 filename = "rustfmt-{}-{}{}".format(
 493                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 494                 )
 495                 self._download_component_helper(
 496                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 497                 )
 498                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 499                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 500                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 501                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 502
 503         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 504         if self.downloading_llvm() and stage0:
 505             # We want the most recent LLVM submodule update to avoid downloading
 506             # LLVM more often than necessary.
 507             #
 508             # This git command finds that commit SHA, looking for bors-authored
 509             # commits that modified src/llvm-project or other relevant version
 510             # stamp files.
 511             #
 512             # This works even in a repository that has not yet initialized
 513             # submodules.
 514             top_level = subprocess.check_output([
 515                 "git", "rev-parse", "--show-toplevel",
 516             ]).decode(sys.getdefaultencoding()).strip()
 517             llvm_sha = subprocess.check_output([
 518                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 519                 "--first-parent", "HEAD",
 520                 "--",
 521                 "{}/src/llvm-project".format(top_level),
 522                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 523                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 524                 "{}/src/version".format(top_level)
 525             ]).decode(sys.getdefaultencoding()).strip()
 526             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 527             llvm_root = self.llvm_root()
 528             llvm_lib = os.path.join(llvm_root, "lib")
 529             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 530                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 531                 for binary in ["llvm-config", "FileCheck"]:
 532                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 533                 for lib in os.listdir(llvm_lib):
 534                     if lib.endswith(".so"):
 535                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 536                 with output(self.llvm_stamp()) as llvm_stamp:
 537                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 538
 539     def downloading_llvm(self):
 540         opt = self.get_toml('download-ci-llvm', 'llvm')
 541         # This is currently all tier 1 targets and tier 2 targets with host tools
 542         # (since others may not have CI artifacts)
 543         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 544         supported_platforms = [
 545             # tier 1
 546             "aarch64-unknown-linux-gnu",
 547             "i686-pc-windows-gnu",
 548             "i686-pc-windows-msvc",
 549             "i686-unknown-linux-gnu",
 550             "x86_64-unknown-linux-gnu",
 551             "x86_64-apple-darwin",
 552             "x86_64-pc-windows-gnu",
 553             "x86_64-pc-windows-msvc",
 554             # tier 2 with host tools
 555             "aarch64-apple-darwin",
 556             "aarch64-pc-windows-msvc",
 557             "aarch64-unknown-linux-musl",
 558             "arm-unknown-linux-gnueabi",
 559             "arm-unknown-linux-gnueabihf",
 560             "armv7-unknown-linux-gnueabihf",
 561             "mips-unknown-linux-gnu",
 562             "mips64-unknown-linux-gnuabi64",
 563             "mips64el-unknown-linux-gnuabi64",
 564             "mipsel-unknown-linux-gnu",
 565             "powerpc-unknown-linux-gnu",
 566             "powerpc64-unknown-linux-gnu",
 567             "powerpc64le-unknown-linux-gnu",
 568             "riscv64gc-unknown-linux-gnu",
 569             "s390x-unknown-linux-gnu",
 570             "x86_64-unknown-freebsd",
 571             "x86_64-unknown-illumos",
 572             "x86_64-unknown-linux-musl",
 573             "x86_64-unknown-netbsd",
 574         ]
 575         return opt == "true" \
 576             or (opt == "if-available" and self.build in supported_platforms)
 577
 578     def _download_component_helper(
 579         self, filename, pattern, tarball_suffix, stage0=True, key=None
 580     ):
 581         if key is None:
 582             if stage0:
 583                 key = self.stage0_compiler.date
 584             else:
 585                 key = self.rustc_commit
 586         cache_dst = os.path.join(self.build_dir, "cache")
 587         rustc_cache = os.path.join(cache_dst, key)
 588         if not os.path.exists(rustc_cache):
 589             os.makedirs(rustc_cache)
 590
 591         if stage0:
 592             base = self._download_url
 593             url = "dist/{}".format(key)
 594         else:
 595             base = "https://ci-artifacts.rust-lang.org"
 596             url = "rustc-builds/{}".format(self.rustc_commit)
 597         tarball = os.path.join(rustc_cache, filename)
 598         if not os.path.exists(tarball):
 599             get(
 600                 base,
 601                 "{}/{}".format(url, filename),
 602                 tarball,
 603                 self.checksums_sha256,
 604                 verbose=self.verbose,
 605                 do_verify=stage0,
 606             )
 607         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 608
 609     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 610         if not llvm_sha:
 611             print("error: could not find commit hash for downloading LLVM")
 612             print("help: maybe your repository history is too shallow?")
 613             print("help: consider disabling `download-ci-llvm`")
 614             print("help: or fetch enough history to include one upstream commit")
 615             exit(1)
 616         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 617         cache_dst = os.path.join(self.build_dir, "cache")
 618         rustc_cache = os.path.join(cache_dst, cache_prefix)
 619         if not os.path.exists(rustc_cache):
 620             os.makedirs(rustc_cache)
 621
 622         base = "https://ci-artifacts.rust-lang.org"
 623         url = "rustc-builds/{}".format(llvm_sha)
 624         if llvm_assertions:
 625             url = url.replace('rustc-builds', 'rustc-builds-alt')
 626         # ci-artifacts are only stored as .xz, not .gz
 627         if not support_xz():
 628             print("error: XZ support is required to download LLVM")
 629             print("help: consider disabling `download-ci-llvm` or using python3")
 630             exit(1)
 631         tarball_suffix = '.tar.xz'
 632         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 633         tarball = os.path.join(rustc_cache, filename)
 634         if not os.path.exists(tarball):
 635             help_on_error = "error: failed to download llvm from ci"
 636             help_on_error += "\nhelp: old builds get deleted after a certain time"
 637             help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
 638             help_on_error += " disable `download-ci-llvm` in config.toml:"
 639             help_on_error += "\n"
 640             help_on_error += "\n[llvm]"
 641             help_on_error += "\ndownload-ci-llvm = false"
 642             help_on_error += "\n"
 643             get(
 644                 base,
 645                 "{}/{}".format(url, filename),
 646                 tarball,
 647                 self.checksums_sha256,
 648                 verbose=self.verbose,
 649                 do_verify=False,
 650                 help_on_error=help_on_error,
 651             )
 652         unpack(tarball, tarball_suffix, self.llvm_root(),
 653                 match="rust-dev",
 654                 verbose=self.verbose)
 655
 656     def fix_bin_or_dylib(self, fname):
 657         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 658         or the RPATH section, to fix the dynamic library search path
 659
 660         This method is only required on NixOS and uses the PatchELF utility to
 661         change the interpreter/RPATH of ELF executables.
 662
 663         Please see https://nixos.org/patchelf.html for more information
 664         """
 665         default_encoding = sys.getdefaultencoding()
 666         try:
 667             ostype = subprocess.check_output(
 668                 ['uname', '-s']).strip().decode(default_encoding)
 669         except subprocess.CalledProcessError:
 670             return
 671         except OSError as reason:
 672             if getattr(reason, 'winerror', None) is not None:
 673                 return
 674             raise reason
 675
 676         if ostype != "Linux":
 677             return
 678
 679         # If the user has asked binaries to be patched for Nix, then
 680         # don't check for NixOS or `/lib`, just continue to the patching.
 681         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 682             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 683             # The latter one does not exist on NixOS when using tmpfs as root.
 684             try:
 685                 with open("/etc/os-release", "r") as f:
 686                     if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
 687                         return
 688             except FileNotFoundError:
 689                 return
 690             if os.path.exists("/lib"):
 691                 return
 692
 693         # At this point we're pretty sure the user is running NixOS or
 694         # using Nix
 695         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 696         print(nix_os_msg, fname)
 697
 698         # Only build `.nix-deps` once.
 699         nix_deps_dir = self.nix_deps_dir
 700         if not nix_deps_dir:
 701             # Run `nix-build` to "build" each dependency (which will likely reuse
 702             # the existing `/nix/store` copy, or at most download a pre-built copy).
 703             #
 704             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 705             # directory, but still reference the actual `/nix/store` path in the rpath
 706             # as it makes it significantly more robust against changes to the location of
 707             # the `.nix-deps` location.
 708             #
 709             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 710             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 711             # patchelf: Needed for patching ELF binaries (see doc comment above).
 712             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 713             nix_expr = '''
 714             with (import <nixpkgs> {});
 715             symlinkJoin {
 716               name = "rust-stage0-dependencies";
 717               paths = [
 718                 zlib
 719                 patchelf
 720                 stdenv.cc.bintools
 721               ];
 722             }
 723             '''
 724             try:
 725                 subprocess.check_output([
 726                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 727                 ])
 728             except subprocess.CalledProcessError as reason:
 729                 print("warning: failed to call nix-build:", reason)
 730                 return
 731             self.nix_deps_dir = nix_deps_dir
 732
 733         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 734         rpath_entries = [
 735             # Relative default, all binary and dynamic libraries we ship
 736             # appear to have this (even when `../lib` is redundant).
 737             "$ORIGIN/../lib",
 738             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 739         ]
 740         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 741         if not fname.endswith(".so"):
 742             # Finally, set the corret .interp for binaries
 743             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 744                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 745
 746         try:
 747             subprocess.check_output([patchelf] + patchelf_args + [fname])
 748         except subprocess.CalledProcessError as reason:
 749             print("warning: failed to call patchelf:", reason)
 750             return
 751
 752     # If `download-rustc` is set, download the most recent commit with CI artifacts
 753     def maybe_download_ci_toolchain(self):
 754         # If `download-rustc` is not set, default to rebuilding.
 755         download_rustc = self.get_toml("download-rustc", section="rust")
 756         if download_rustc is None or download_rustc == "false":
 757             return None
 758         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 759
 760         # Handle running from a directory other than the top level
 761         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 762         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 763         compiler = "{}/compiler/".format(top_level)
 764         library = "{}/library/".format(top_level)
 765
 766         # Look for a version to compare to based on the current commit.
 767         # Only commits merged by bors will have CI artifacts.
 768         merge_base = [
 769             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 770             "--first-parent", "HEAD"
 771         ]
 772         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 773         if not commit:
 774             print("error: could not find commit hash for downloading rustc")
 775             print("help: maybe your repository history is too shallow?")
 776             print("help: consider disabling `download-rustc`")
 777             print("help: or fetch enough history to include one upstream commit")
 778             exit(1)
 779
 780         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 781         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 782         if status != 0:
 783             if download_rustc == "if-unchanged":
 784                 if self.verbose:
 785                     print("warning: saw changes to compiler/ or library/ since {}; " \
 786                           "ignoring `download-rustc`".format(commit))
 787                 return None
 788             print("warning: `download-rustc` is enabled, but there are changes to " \
 789                   "compiler/ or library/")
 790
 791         if self.verbose:
 792             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
 793         self.rustc_commit = commit
 794         # FIXME: support downloading artifacts from the beta channel
 795         self.download_toolchain(False, "nightly")
 796
 797     def rustc_stamp(self, stage0):
 798         """Return the path for .rustc-stamp at the given stage
 799
 800         >>> rb = RustBuild()
 801         >>> rb.build_dir = "build"
 802         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 803         True
 804         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 805         True
 806         """
 807         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 808
 809     def rustfmt_stamp(self):
 810         """Return the path for .rustfmt-stamp
 811
 812         >>> rb = RustBuild()
 813         >>> rb.build_dir = "build"
 814         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 815         True
 816         """
 817         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 818
 819     def llvm_stamp(self):
 820         """Return the path for .llvm-stamp
 821
 822         >>> rb = RustBuild()
 823         >>> rb.build_dir = "build"
 824         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 825         True
 826         """
 827         return os.path.join(self.llvm_root(), '.llvm-stamp')
 828
 829
 830     def program_out_of_date(self, stamp_path, key):
 831         """Check if the given program stamp is out of date"""
 832         if not os.path.exists(stamp_path) or self.clean:
 833             return True
 834         with open(stamp_path, 'r') as stamp:
 835             return key != stamp.read()
 836
 837     def bin_root(self, stage0):
 838         """Return the binary root directory for the given stage
 839
 840         >>> rb = RustBuild()
 841         >>> rb.build_dir = "build"
 842         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 843         True
 844         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 845         True
 846
 847         When the 'build' property is given should be a nested directory:
 848
 849         >>> rb.build = "devel"
 850         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 851         True
 852         """
 853         if stage0:
 854             subdir = "stage0"
 855         else:
 856             subdir = "ci-rustc"
 857         return os.path.join(self.build_dir, self.build, subdir)
 858
 859     def llvm_root(self):
 860         """Return the CI LLVM root directory
 861
 862         >>> rb = RustBuild()
 863         >>> rb.build_dir = "build"
 864         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 865         True
 866
 867         When the 'build' property is given should be a nested directory:
 868
 869         >>> rb.build = "devel"
 870         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 871         True
 872         """
 873         return os.path.join(self.build_dir, self.build, "ci-llvm")
 874
 875     def get_toml(self, key, section=None):
 876         """Returns the value of the given key in config.toml, otherwise returns None
 877
 878         >>> rb = RustBuild()
 879         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 880         >>> rb.get_toml("key2")
 881         'value2'
 882
 883         If the key does not exist, the result is None:
 884
 885         >>> rb.get_toml("key3") is None
 886         True
 887
 888         Optionally also matches the section the key appears in
 889
 890         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 891         >>> rb.get_toml('key', 'a')
 892         'value1'
 893         >>> rb.get_toml('key', 'b')
 894         'value2'
 895         >>> rb.get_toml('key', 'c') is None
 896         True
 897
 898         >>> rb.config_toml = 'key1 = true'
 899         >>> rb.get_toml("key1")
 900         'true'
 901         """
 902
 903         cur_section = None
 904         for line in self.config_toml.splitlines():
 905             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 906             if section_match is not None:
 907                 cur_section = section_match.group(1)
 908
 909             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 910             if match is not None:
 911                 value = match.group(1)
 912                 if section is None or section == cur_section:
 913                     return self.get_string(value) or value.strip()
 914         return None
 915
 916     def cargo(self):
 917         """Return config path for cargo"""
 918         return self.program_config('cargo')
 919
 920     def rustc(self, stage0):
 921         """Return config path for rustc"""
 922         return self.program_config('rustc', stage0)
 923
 924     def rustfmt(self):
 925         """Return config path for rustfmt"""
 926         if self.stage0_rustfmt is None:
 927             return None
 928         return self.program_config('rustfmt')
 929
 930     def program_config(self, program, stage0=True):
 931         """Return config path for the given program at the given stage
 932
 933         >>> rb = RustBuild()
 934         >>> rb.config_toml = 'rustc = "rustc"\\n'
 935         >>> rb.program_config('rustc')
 936         'rustc'
 937         >>> rb.config_toml = ''
 938         >>> cargo_path = rb.program_config('cargo', True)
 939         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 940         ... "bin", "cargo")
 941         True
 942         >>> cargo_path = rb.program_config('cargo', False)
 943         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 944         ... "bin", "cargo")
 945         True
 946         """
 947         config = self.get_toml(program)
 948         if config:
 949             return os.path.expanduser(config)
 950         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 951             program, self.exe_suffix()))
 952
 953     @staticmethod
 954     def get_string(line):
 955         """Return the value between double quotes
 956
 957         >>> RustBuild.get_string('    "devel"   ')
 958         'devel'
 959         >>> RustBuild.get_string("    'devel'   ")
 960         'devel'
 961         >>> RustBuild.get_string('devel') is None
 962         True
 963         >>> RustBuild.get_string('    "devel   ')
 964         ''
 965         """
 966         start = line.find('"')
 967         if start != -1:
 968             end = start + 1 + line[start + 1:].find('"')
 969             return line[start + 1:end]
 970         start = line.find('\'')
 971         if start != -1:
 972             end = start + 1 + line[start + 1:].find('\'')
 973             return line[start + 1:end]
 974         return None
 975
 976     @staticmethod
 977     def exe_suffix():
 978         """Return a suffix for executables"""
 979         if sys.platform == 'win32':
 980             return '.exe'
 981         return ''
 982
 983     def bootstrap_binary(self):
 984         """Return the path of the bootstrap binary
 985
 986         >>> rb = RustBuild()
 987         >>> rb.build_dir = "build"
 988         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 989         ... "debug", "bootstrap")
 990         True
 991         """
 992         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 993
 994     def build_bootstrap(self):
 995         """Build bootstrap"""
 996         print("Building rustbuild")
 997         build_dir = os.path.join(self.build_dir, "bootstrap")
 998         if self.clean and os.path.exists(build_dir):
 999             shutil.rmtree(build_dir)
1000         env = os.environ.copy()
1001         # `CARGO_BUILD_TARGET` breaks bootstrap build.
1002         # See also: <https://github.com/rust-lang/rust/issues/70208>.
1003         if "CARGO_BUILD_TARGET" in env:
1004             del env["CARGO_BUILD_TARGET"]
1005         env["CARGO_TARGET_DIR"] = build_dir
1006         env["RUSTC"] = self.rustc(True)
1007         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1008             (os.pathsep + env["LD_LIBRARY_PATH"]) \
1009             if "LD_LIBRARY_PATH" in env else ""
1010         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1011             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
1012             if "DYLD_LIBRARY_PATH" in env else ""
1013         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1014             (os.pathsep + env["LIBRARY_PATH"]) \
1015             if "LIBRARY_PATH" in env else ""
1016
1017         # preserve existing RUSTFLAGS
1018         env.setdefault("RUSTFLAGS", "")
1019         build_section = "target.{}".format(self.build)
1020         target_features = []
1021         if self.get_toml("crt-static", build_section) == "true":
1022             target_features += ["+crt-static"]
1023         elif self.get_toml("crt-static", build_section) == "false":
1024             target_features += ["-crt-static"]
1025         if target_features:
1026             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1027         target_linker = self.get_toml("linker", build_section)
1028         if target_linker is not None:
1029             env["RUSTFLAGS"] += " -C linker=" + target_linker
1030         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1031         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1032         if self.get_toml("deny-warnings", "rust") != "false":
1033             env["RUSTFLAGS"] += " -Dwarnings"
1034
1035         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1036             os.pathsep + env["PATH"]
1037         if not os.path.isfile(self.cargo()):
1038             raise Exception("no cargo executable found at `{}`".format(
1039                 self.cargo()))
1040         args = [self.cargo(), "build", "--manifest-path",
1041                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1042         for _ in range(0, self.verbose):
1043             args.append("--verbose")
1044         if self.use_locked_deps:
1045             args.append("--locked")
1046         if self.use_vendored_sources:
1047             args.append("--frozen")
1048         run(args, env=env, verbose=self.verbose)
1049
1050     def build_triple(self):
1051         """Build triple as in LLVM
1052
1053         Note that `default_build_triple` is moderately expensive,
1054         so use `self.build` where possible.
1055         """
1056         config = self.get_toml('build')
1057         if config:
1058             return config
1059         return default_build_triple(self.verbose)
1060
1061     def check_submodule(self, module, slow_submodules):
1062         if not slow_submodules:
1063             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1064                                            cwd=os.path.join(self.rust_root, module),
1065                                            stdout=subprocess.PIPE)
1066             return checked_out
1067         else:
1068             return None
1069
1070     def update_submodule(self, module, checked_out, recorded_submodules):
1071         module_path = os.path.join(self.rust_root, module)
1072
1073         if checked_out is not None:
1074             default_encoding = sys.getdefaultencoding()
1075             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1076             if recorded_submodules[module] == checked_out:
1077                 return
1078
1079         print("Updating submodule", module)
1080
1081         run(["git", "submodule", "-q", "sync", module],
1082             cwd=self.rust_root, verbose=self.verbose)
1083
1084         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1085         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1086             update_args.append("--progress")
1087         update_args.append(module)
1088         try:
1089             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1090         except RuntimeError:
1091             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1092             print('Either stash the changes by running "git stash" within the submodule\'s')
1093             print('directory, reset them by running "git reset --hard", or commit them.')
1094             print("To reset all submodules' changes run", end=" ")
1095             print('"git submodule foreach --recursive git reset --hard".')
1096             raise SystemExit(1)
1097
1098         run(["git", "reset", "-q", "--hard"],
1099             cwd=module_path, verbose=self.verbose)
1100         run(["git", "clean", "-qdfx"],
1101             cwd=module_path, verbose=self.verbose)
1102
1103     def update_submodules(self):
1104         """Update submodules"""
1105         has_git = os.path.exists(os.path.join(self.rust_root, ".git"))
1106         # This just arbitrarily checks for cargo, but any workspace member in
1107         # a submodule would work.
1108         has_submodules = os.path.exists(os.path.join(self.rust_root, "src/tools/cargo/Cargo.toml"))
1109         if not has_git and not has_submodules:
1110             print("This is not a git repository, and the requisite git submodules were not found.")
1111             print("If you downloaded the source from https://github.com/rust-lang/rust/releases,")
1112             print("those sources will not work. Instead, consider downloading from the source")
1113             print("releases linked at")
1114             print("https://forge.rust-lang.org/infra/other-installation-methods.html#source-code")
1115             print("or clone the repository at https://github.com/rust-lang/rust/.")
1116             raise SystemExit(1)
1117         if not has_git or self.get_toml('submodules') == "false":
1118             return
1119
1120         default_encoding = sys.getdefaultencoding()
1121
1122         # check the existence and version of 'git' command
1123         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1124         self.git_version = distutils.version.LooseVersion(git_version_str)
1125
1126         slow_submodules = self.get_toml('fast-submodules') == "false"
1127         start_time = time()
1128         if slow_submodules:
1129             print('Unconditionally updating submodules')
1130         else:
1131             print('Updating only changed submodules')
1132         default_encoding = sys.getdefaultencoding()
1133         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1134         # currently requires everything in a workspace to be "locally present" when starting a
1135         # build, and will give a hard error if any Cargo.toml files are missing.
1136         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1137         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1138         #   That will still require cloning the submodules the second you check the standard
1139         #   library, though...
1140         # FIXME: Is there a way to avoid hard-coding the submodules required?
1141         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1142         submodules = [
1143             "src/tools/rust-installer",
1144             "src/tools/cargo",
1145             "src/tools/rls",
1146             "src/tools/miri",
1147             "library/backtrace",
1148             "library/stdarch"
1149         ]
1150         filtered_submodules = []
1151         submodules_names = []
1152         for module in submodules:
1153             check = self.check_submodule(module, slow_submodules)
1154             filtered_submodules.append((module, check))
1155             submodules_names.append(module)
1156         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1157                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1158         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1159         # { filename: hash }
1160         recorded_submodules = {}
1161         for data in recorded:
1162             # [mode, kind, hash, filename]
1163             data = data.split()
1164             recorded_submodules[data[3]] = data[2]
1165         for module in filtered_submodules:
1166             self.update_submodule(module[0], module[1], recorded_submodules)
1167         print("  Submodules updated in %.2f seconds" % (time() - start_time))
1168
1169     def set_dist_environment(self, url):
1170         """Set download URL for normal environment"""
1171         if 'RUSTUP_DIST_SERVER' in os.environ:
1172             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1173         else:
1174             self._download_url = url
1175
1176     def check_vendored_status(self):
1177         """Check that vendoring is configured properly"""
1178         vendor_dir = os.path.join(self.rust_root, 'vendor')
1179         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1180             if os.getuid() == 0:
1181                 self.use_vendored_sources = True
1182                 print('info: looks like you\'re trying to run this command as root')
1183                 print('      and so in order to preserve your $HOME this will now')
1184                 print('      use vendored sources by default.')
1185                 if not os.path.exists(vendor_dir):
1186                     print('error: vendoring required, but vendor directory does not exist.')
1187                     print('       Run `cargo vendor` without sudo to initialize the '
1188                           'vendor directory.')
1189                     raise Exception("{} not found".format(vendor_dir))
1190
1191         if self.use_vendored_sources:
1192             config = ("[source.crates-io]\n"
1193                       "replace-with = 'vendored-sources'\n"
1194                       "registry = 'https://example.com'\n"
1195                       "\n"
1196                       "[source.vendored-sources]\n"
1197                       "directory = '{}/vendor'\n"
1198                       .format(self.rust_root))
1199             if not os.path.exists('.cargo'):
1200                 os.makedirs('.cargo')
1201                 with output('.cargo/config') as cargo_config:
1202                     cargo_config.write(config)
1203             else:
1204                 print('info: using vendored source, but .cargo/config is already present.')
1205                 print('      Reusing the current configuration file. But you may want to '
1206                       'configure vendoring like this:')
1207                 print(config)
1208         else:
1209             if os.path.exists('.cargo'):
1210                 shutil.rmtree('.cargo')
1211
1212     def ensure_vendored(self):
1213         """Ensure that the vendored sources are available if needed"""
1214         vendor_dir = os.path.join(self.rust_root, 'vendor')
1215         # Note that this does not handle updating the vendored dependencies if
1216         # the rust git repository is updated. Normal development usually does
1217         # not use vendoring, so hopefully this isn't too much of a problem.
1218         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1219             run([
1220                 self.cargo(),
1221                 "vendor",
1222                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1223                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1224             ], verbose=self.verbose, cwd=self.rust_root)
1225
1226
1227 def bootstrap(help_triggered):
1228     """Configure, fetch, build and run the initial bootstrap"""
1229
1230     # If the user is asking for help, let them know that the whole download-and-build
1231     # process has to happen before anything is printed out.
1232     if help_triggered:
1233         print("info: Downloading and building bootstrap before processing --help")
1234         print("      command. See src/bootstrap/README.md for help with common")
1235         print("      commands.")
1236
1237     parser = argparse.ArgumentParser(description='Build rust')
1238     parser.add_argument('--config')
1239     parser.add_argument('--build')
1240     parser.add_argument('--clean', action='store_true')
1241     parser.add_argument('-v', '--verbose', action='count', default=0)
1242
1243     args = [a for a in sys.argv if a != '-h' and a != '--help']
1244     args, _ = parser.parse_known_args(args)
1245
1246     # Configure initial bootstrap
1247     build = RustBuild()
1248     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1249     build.verbose = args.verbose
1250     build.clean = args.clean
1251
1252     # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1253     # then `config.toml` in the root directory.
1254     toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1255     using_default_path = toml_path is None
1256     if using_default_path:
1257         toml_path = 'config.toml'
1258         if not os.path.exists(toml_path):
1259             toml_path = os.path.join(build.rust_root, toml_path)
1260
1261     # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1262     # but not if `config.toml` hasn't been created.
1263     if not using_default_path or os.path.exists(toml_path):
1264         with open(toml_path) as config:
1265             build.config_toml = config.read()
1266
1267     profile = build.get_toml('profile')
1268     if profile is not None:
1269         include_file = 'config.{}.toml'.format(profile)
1270         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1271         include_path = os.path.join(include_dir, include_file)
1272         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1273         # specific key, so appending our defaults at the end allows the user to override them
1274         with open(include_path) as included_toml:
1275             build.config_toml += os.linesep + included_toml.read()
1276
1277     config_verbose = build.get_toml('verbose', 'build')
1278     if config_verbose is not None:
1279         build.verbose = max(build.verbose, int(config_verbose))
1280
1281     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1282
1283     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1284
1285     build.check_vendored_status()
1286
1287     build_dir = build.get_toml('build-dir', 'build') or 'build'
1288     build.build_dir = os.path.abspath(build_dir)
1289
1290     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1291         data = json.load(f)
1292     build.checksums_sha256 = data["checksums_sha256"]
1293     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1294     if data.get("rustfmt") is not None:
1295         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1296
1297     build.set_dist_environment(data["dist_server"])
1298
1299     build.build = args.build or build.build_triple()
1300
1301     # Acquire the lock before doing any build actions
1302     # The lock is released when `lock` is dropped
1303     if not os.path.exists(build.build_dir):
1304         os.makedirs(build.build_dir)
1305     lock = acquire_lock(build.build_dir)
1306     build.update_submodules()
1307
1308     # Fetch/build the bootstrap
1309     build.download_toolchain()
1310     # Download the master compiler if `download-rustc` is set
1311     build.maybe_download_ci_toolchain()
1312     sys.stdout.flush()
1313     build.ensure_vendored()
1314     build.build_bootstrap()
1315     sys.stdout.flush()
1316
1317     # Run the bootstrap
1318     args = [build.bootstrap_binary()]
1319     args.extend(sys.argv[1:])
1320     env = os.environ.copy()
1321     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1322     env["BOOTSTRAP_PYTHON"] = sys.executable
1323     env["RUSTC_BOOTSTRAP"] = '1'
1324     if build.rustc_commit is not None:
1325         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1326     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1327
1328
1329 def main():
1330     """Entry point for the bootstrap process"""
1331     start_time = time()
1332
1333     # x.py help <cmd> ...
1334     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1335         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1336
1337     help_triggered = (
1338         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1339     try:
1340         bootstrap(help_triggered)
1341         if not help_triggered:
1342             print("Build completed successfully in {}".format(
1343                 format_build_time(time() - start_time)))
1344     except (SystemExit, KeyboardInterrupt) as error:
1345         if hasattr(error, 'code') and isinstance(error.code, int):
1346             exit_code = error.code
1347         else:
1348             exit_code = 1
1349             print(error)
1350         if not help_triggered:
1351             print("Build completed unsuccessfully in {}".format(
1352                 format_build_time(time() - start_time)))
1353         sys.exit(exit_code)
1354
1355
1356 if __name__ == '__main__':
1357     main()