src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time
  17
  18 # Acquire a lock on the build directory to make sure that
  19 # we don't cause a race condition while building
  20 # Lock is created in `build_dir/lock.db`
  21 def acquire_lock(build_dir):
  22     try:
  23         import sqlite3
  24
  25         path = os.path.join(build_dir, "lock.db")
  26         try:
  27             con = sqlite3.Connection(path, timeout=0)
  28             curs = con.cursor()
  29             curs.execute("BEGIN EXCLUSIVE")
  30             # The lock is released when the cursor is dropped
  31             return curs
  32         # If the database is busy then lock has already been acquired
  33         # so we wait for the lock.
  34         # We retry every quarter second so that execution is passed back to python
  35         # so that it can handle signals
  36         except sqlite3.OperationalError:
  37             del con
  38             del curs
  39             print("Waiting for lock on build directory")
  40             con = sqlite3.Connection(path, timeout=0.25)
  41             curs = con.cursor()
  42             while True:
  43                 try:
  44                     curs.execute("BEGIN EXCLUSIVE")
  45                 except sqlite3.OperationalError:
  46                     pass
  47             return curs
  48     except ImportError:
  49         print("warning: sqlite3 not available in python, skipping build directory lock")
  50         print("please file an issue on rust-lang/rust")
  51         print("this is not a problem for non-concurrent x.py invocations")
  52         return None
  53
  54 def support_xz():
  55     try:
  56         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  57             temp_path = temp_file.name
  58         with tarfile.open(temp_path, "w:xz"):
  59             pass
  60         return True
  61     except tarfile.CompressionError:
  62         return False
  63
  64 def get(base, url, path, checksums, verbose=False, do_verify=True):
  65     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  66         temp_path = temp_file.name
  67
  68     try:
  69         if do_verify:
  70             if url not in checksums:
  71                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  72             sha256 = checksums[url]
  73             if os.path.exists(path):
  74                 if verify(path, sha256, False):
  75                     if verbose:
  76                         print("using already-download file", path)
  77                     return
  78                 else:
  79                     if verbose:
  80                         print("ignoring already-download file",
  81                             path, "due to failed verification")
  82                     os.unlink(path)
  83         download(temp_path, "{}/{}".format(base, url), True, verbose)
  84         if do_verify and not verify(temp_path, sha256, verbose):
  85             raise RuntimeError("failed verification")
  86         if verbose:
  87             print("moving {} to {}".format(temp_path, path))
  88         shutil.move(temp_path, path)
  89     finally:
  90         if os.path.isfile(temp_path):
  91             if verbose:
  92                 print("removing", temp_path)
  93             os.unlink(temp_path)
  94
  95
  96 def download(path, url, probably_big, verbose):
  97     for _ in range(0, 4):
  98         try:
  99             _download(path, url, probably_big, verbose, True)
 100             return
 101         except RuntimeError:
 102             print("\nspurious failure, trying again")
 103     _download(path, url, probably_big, verbose, False)
 104
 105
 106 def _download(path, url, probably_big, verbose, exception):
 107     if probably_big or verbose:
 108         print("downloading {}".format(url))
 109     # see https://serverfault.com/questions/301128/how-to-download
 110     if sys.platform == 'win32':
 111         run(["PowerShell.exe", "/nologo", "-Command",
 112              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 113              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
 114             verbose=verbose,
 115             exception=exception)
 116     else:
 117         if probably_big or verbose:
 118             option = "-#"
 119         else:
 120             option = "-s"
 121         require(["curl", "--version"])
 122         run(["curl", option,
 123              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
 124              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
 125              "--retry", "3", "-Sf", "-o", path, url],
 126             verbose=verbose,
 127             exception=exception)
 128
 129
 130 def verify(path, expected, verbose):
 131     """Check if the sha256 sum of the given path is valid"""
 132     if verbose:
 133         print("verifying", path)
 134     with open(path, "rb") as source:
 135         found = hashlib.sha256(source.read()).hexdigest()
 136     verified = found == expected
 137     if not verified:
 138         print("invalid checksum:\n"
 139               "    found:    {}\n"
 140               "    expected: {}".format(found, expected))
 141     return verified
 142
 143
 144 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 145     """Unpack the given tarball file"""
 146     print("extracting", tarball)
 147     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 148     with contextlib.closing(tarfile.open(tarball)) as tar:
 149         for member in tar.getnames():
 150             if "/" not in member:
 151                 continue
 152             name = member.replace(fname + "/", "", 1)
 153             if match is not None and not name.startswith(match):
 154                 continue
 155             name = name[len(match) + 1:]
 156
 157             dst_path = os.path.join(dst, name)
 158             if verbose:
 159                 print("  extracting", member)
 160             tar.extract(member, dst)
 161             src_path = os.path.join(dst, member)
 162             if os.path.isdir(src_path) and os.path.exists(dst_path):
 163                 continue
 164             shutil.move(src_path, dst_path)
 165     shutil.rmtree(os.path.join(dst, fname))
 166
 167
 168 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
 169     """Run a child program in a new process"""
 170     if verbose:
 171         print("running: " + ' '.join(args))
 172     sys.stdout.flush()
 173     # Use Popen here instead of call() as it apparently allows powershell on
 174     # Windows to not lock up waiting for input presumably.
 175     ret = subprocess.Popen(args, **kwargs)
 176     code = ret.wait()
 177     if code != 0:
 178         err = "failed to run: " + ' '.join(args)
 179         if verbose or exception:
 180             raise RuntimeError(err)
 181         # For most failures, we definitely do want to print this error, or the user will have no
 182         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 183         # have already printed an error above, so there's no need to print the exact command we're
 184         # running.
 185         if is_bootstrap:
 186             sys.exit(1)
 187         else:
 188             sys.exit(err)
 189
 190
 191 def require(cmd, exit=True):
 192     '''Run a command, returning its output.
 193     On error,
 194         If `exit` is `True`, exit the process.
 195         Otherwise, return None.'''
 196     try:
 197         return subprocess.check_output(cmd).strip()
 198     except (subprocess.CalledProcessError, OSError) as exc:
 199         if not exit:
 200             return None
 201         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 202         print("Please make sure it's installed and in the path.")
 203         sys.exit(1)
 204
 205
 206 def format_build_time(duration):
 207     """Return a nicer format for build time
 208
 209     >>> format_build_time('300')
 210     '0:05:00'
 211     """
 212     return str(datetime.timedelta(seconds=int(duration)))
 213
 214
 215 def default_build_triple(verbose):
 216     """Build triple as in LLVM"""
 217     # If the user already has a host build triple with an existing `rustc`
 218     # install, use their preference. This fixes most issues with Windows builds
 219     # being detected as GNU instead of MSVC.
 220     default_encoding = sys.getdefaultencoding()
 221     try:
 222         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 223                 stderr=subprocess.DEVNULL)
 224         version = version.decode(default_encoding)
 225         host = next(x for x in version.split('\n') if x.startswith("host: "))
 226         triple = host.split("host: ")[1]
 227         if verbose:
 228             print("detected default triple {} from pre-installed rustc".format(triple))
 229         return triple
 230     except Exception as e:
 231         if verbose:
 232             print("pre-installed rustc not detected: {}".format(e))
 233             print("falling back to auto-detect")
 234
 235     required = sys.platform != 'win32'
 236     ostype = require(["uname", "-s"], exit=required)
 237     cputype = require(['uname', '-m'], exit=required)
 238
 239     # If we do not have `uname`, assume Windows.
 240     if ostype is None or cputype is None:
 241         return 'x86_64-pc-windows-msvc'
 242
 243     ostype = ostype.decode(default_encoding)
 244     cputype = cputype.decode(default_encoding)
 245
 246     # The goal here is to come up with the same triple as LLVM would,
 247     # at least for the subset of platforms we're willing to target.
 248     ostype_mapper = {
 249         'Darwin': 'apple-darwin',
 250         'DragonFly': 'unknown-dragonfly',
 251         'FreeBSD': 'unknown-freebsd',
 252         'Haiku': 'unknown-haiku',
 253         'NetBSD': 'unknown-netbsd',
 254         'OpenBSD': 'unknown-openbsd'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if ostype in ostype_mapper:
 259         ostype = ostype_mapper[ostype]
 260     elif ostype == 'Linux':
 261         os_from_sp = subprocess.check_output(
 262             ['uname', '-o']).strip().decode(default_encoding)
 263         if os_from_sp == 'Android':
 264             ostype = 'linux-android'
 265         else:
 266             ostype = 'unknown-linux-gnu'
 267     elif ostype == 'SunOS':
 268         ostype = 'pc-solaris'
 269         # On Solaris, uname -m will return a machine classification instead
 270         # of a cpu type, so uname -p is recommended instead.  However, the
 271         # output from that option is too generic for our purposes (it will
 272         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 273         # must be used instead.
 274         cputype = require(['isainfo', '-k']).decode(default_encoding)
 275         # sparc cpus have sun as a target vendor
 276         if 'sparc' in cputype:
 277             ostype = 'sun-solaris'
 278     elif ostype.startswith('MINGW'):
 279         # msys' `uname` does not print gcc configuration, but prints msys
 280         # configuration. so we cannot believe `uname -m`:
 281         # msys1 is always i686 and msys2 is always x86_64.
 282         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 283         # MINGW64 on x86_64.
 284         ostype = 'pc-windows-gnu'
 285         cputype = 'i686'
 286         if os.environ.get('MSYSTEM') == 'MINGW64':
 287             cputype = 'x86_64'
 288     elif ostype.startswith('MSYS'):
 289         ostype = 'pc-windows-gnu'
 290     elif ostype.startswith('CYGWIN_NT'):
 291         cputype = 'i686'
 292         if ostype.endswith('WOW64'):
 293             cputype = 'x86_64'
 294         ostype = 'pc-windows-gnu'
 295     elif sys.platform == 'win32':
 296         # Some Windows platforms might have a `uname` command that returns a
 297         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 298         # these cases, fall back to using sys.platform.
 299         return 'x86_64-pc-windows-msvc'
 300     else:
 301         err = "unknown OS type: {}".format(ostype)
 302         sys.exit(err)
 303
 304     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
 305         cputype = subprocess.check_output(
 306               ['uname', '-p']).strip().decode(default_encoding)
 307     cputype_mapper = {
 308         'BePC': 'i686',
 309         'aarch64': 'aarch64',
 310         'amd64': 'x86_64',
 311         'arm64': 'aarch64',
 312         'i386': 'i686',
 313         'i486': 'i686',
 314         'i686': 'i686',
 315         'i786': 'i686',
 316         'm68k': 'm68k',
 317         'powerpc': 'powerpc',
 318         'powerpc64': 'powerpc64',
 319         'powerpc64le': 'powerpc64le',
 320         'ppc': 'powerpc',
 321         'ppc64': 'powerpc64',
 322         'ppc64le': 'powerpc64le',
 323         'riscv64': 'riscv64gc',
 324         's390x': 's390x',
 325         'x64': 'x86_64',
 326         'x86': 'i686',
 327         'x86-64': 'x86_64',
 328         'x86_64': 'x86_64'
 329     }
 330
 331     # Consider the direct transformation first and then the special cases
 332     if cputype in cputype_mapper:
 333         cputype = cputype_mapper[cputype]
 334     elif cputype in {'xscale', 'arm'}:
 335         cputype = 'arm'
 336         if ostype == 'linux-android':
 337             ostype = 'linux-androideabi'
 338         elif ostype == 'unknown-freebsd':
 339             cputype = subprocess.check_output(
 340                 ['uname', '-p']).strip().decode(default_encoding)
 341             ostype = 'unknown-freebsd'
 342     elif cputype == 'armv6l':
 343         cputype = 'arm'
 344         if ostype == 'linux-android':
 345             ostype = 'linux-androideabi'
 346         else:
 347             ostype += 'eabihf'
 348     elif cputype in {'armv7l', 'armv8l'}:
 349         cputype = 'armv7'
 350         if ostype == 'linux-android':
 351             ostype = 'linux-androideabi'
 352         else:
 353             ostype += 'eabihf'
 354     elif cputype == 'mips':
 355         if sys.byteorder == 'big':
 356             cputype = 'mips'
 357         elif sys.byteorder == 'little':
 358             cputype = 'mipsel'
 359         else:
 360             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 361     elif cputype == 'mips64':
 362         if sys.byteorder == 'big':
 363             cputype = 'mips64'
 364         elif sys.byteorder == 'little':
 365             cputype = 'mips64el'
 366         else:
 367             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 368         # only the n64 ABI is supported, indicate it
 369         ostype += 'abi64'
 370     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 371         pass
 372     else:
 373         err = "unknown cpu type: {}".format(cputype)
 374         sys.exit(err)
 375
 376     return "{}-{}".format(cputype, ostype)
 377
 378
 379 @contextlib.contextmanager
 380 def output(filepath):
 381     tmp = filepath + '.tmp'
 382     with open(tmp, 'w') as f:
 383         yield f
 384     try:
 385         if os.path.exists(filepath):
 386             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 387     except OSError:
 388         shutil.copy2(tmp, filepath)
 389         os.remove(tmp)
 390         return
 391     os.rename(tmp, filepath)
 392
 393
 394 class Stage0Toolchain:
 395     def __init__(self, stage0_payload):
 396         self.date = stage0_payload["date"]
 397         self.version = stage0_payload["version"]
 398
 399     def channel(self):
 400         return self.version + "-" + self.date
 401
 402
 403 class RustBuild(object):
 404     """Provide all the methods required to build Rust"""
 405     def __init__(self):
 406         self.checksums_sha256 = {}
 407         self.stage0_compiler = None
 408         self.stage0_rustfmt = None
 409         self._download_url = ''
 410         self.build = ''
 411         self.build_dir = ''
 412         self.clean = False
 413         self.config_toml = ''
 414         self.rust_root = ''
 415         self.use_locked_deps = ''
 416         self.use_vendored_sources = ''
 417         self.verbose = False
 418         self.git_version = None
 419         self.nix_deps_dir = None
 420         self.rustc_commit = None
 421
 422     def download_toolchain(self, stage0=True, rustc_channel=None):
 423         """Fetch the build system for Rust, written in Rust
 424
 425         This method will build a cache directory, then it will fetch the
 426         tarball which has the stage0 compiler used to then bootstrap the Rust
 427         compiler itself.
 428
 429         Each downloaded tarball is extracted, after that, the script
 430         will move all the content to the right place.
 431         """
 432         if rustc_channel is None:
 433             rustc_channel = self.stage0_compiler.version
 434         bin_root = self.bin_root(stage0)
 435
 436         key = self.stage0_compiler.date
 437         if not stage0:
 438             key += str(self.rustc_commit)
 439         if self.rustc(stage0).startswith(bin_root) and \
 440                 (not os.path.exists(self.rustc(stage0)) or
 441                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 442             if os.path.exists(bin_root):
 443                 shutil.rmtree(bin_root)
 444             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 445             filename = "rust-std-{}-{}{}".format(
 446                 rustc_channel, self.build, tarball_suffix)
 447             pattern = "rust-std-{}".format(self.build)
 448             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 449             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 450                                               tarball_suffix)
 451             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 452             # download-rustc doesn't need its own cargo, it can just use beta's.
 453             if stage0:
 454                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 455                                                 tarball_suffix)
 456                 self._download_component_helper(filename, "cargo", tarball_suffix)
 457                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 458             else:
 459                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 460                 self._download_component_helper(
 461                     filename, "rustc-dev", tarball_suffix, stage0
 462                 )
 463
 464             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 465             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 466             lib_dir = "{}/lib".format(bin_root)
 467             for lib in os.listdir(lib_dir):
 468                 if lib.endswith(".so"):
 469                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 470             with output(self.rustc_stamp(stage0)) as rust_stamp:
 471                 rust_stamp.write(key)
 472
 473         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 474             not os.path.exists(self.rustfmt())
 475             or self.program_out_of_date(
 476                 self.rustfmt_stamp(),
 477                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 478             )
 479         ):
 480             if self.stage0_rustfmt is not None:
 481                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 482                 filename = "rustfmt-{}-{}{}".format(
 483                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 484                 )
 485                 self._download_component_helper(
 486                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 487                 )
 488                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 489                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 490                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 491                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 492
 493         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 494         if self.downloading_llvm() and stage0:
 495             # We want the most recent LLVM submodule update to avoid downloading
 496             # LLVM more often than necessary.
 497             #
 498             # This git command finds that commit SHA, looking for bors-authored
 499             # commits that modified src/llvm-project or other relevant version
 500             # stamp files.
 501             #
 502             # This works even in a repository that has not yet initialized
 503             # submodules.
 504             top_level = subprocess.check_output([
 505                 "git", "rev-parse", "--show-toplevel",
 506             ]).decode(sys.getdefaultencoding()).strip()
 507             llvm_sha = subprocess.check_output([
 508                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 509                 "--first-parent", "HEAD",
 510                 "--",
 511                 "{}/src/llvm-project".format(top_level),
 512                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 513                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 514                 "{}/src/version".format(top_level)
 515             ]).decode(sys.getdefaultencoding()).strip()
 516             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 517             llvm_root = self.llvm_root()
 518             llvm_lib = os.path.join(llvm_root, "lib")
 519             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 520                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 521                 for binary in ["llvm-config", "FileCheck"]:
 522                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 523                 for lib in os.listdir(llvm_lib):
 524                     if lib.endswith(".so"):
 525                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 526                 with output(self.llvm_stamp()) as llvm_stamp:
 527                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 528
 529     def downloading_llvm(self):
 530         opt = self.get_toml('download-ci-llvm', 'llvm')
 531         # This is currently all tier 1 targets and tier 2 targets with host tools
 532         # (since others may not have CI artifacts)
 533         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 534         supported_platforms = [
 535             # tier 1
 536             "aarch64-unknown-linux-gnu",
 537             "i686-pc-windows-gnu",
 538             "i686-pc-windows-msvc",
 539             "i686-unknown-linux-gnu",
 540             "x86_64-unknown-linux-gnu",
 541             "x86_64-apple-darwin",
 542             "x86_64-pc-windows-gnu",
 543             "x86_64-pc-windows-msvc",
 544             # tier 2 with host tools
 545             "aarch64-apple-darwin",
 546             "aarch64-pc-windows-msvc",
 547             "aarch64-unknown-linux-musl",
 548             "arm-unknown-linux-gnueabi",
 549             "arm-unknown-linux-gnueabihf",
 550             "armv7-unknown-linux-gnueabihf",
 551             "mips-unknown-linux-gnu",
 552             "mips64-unknown-linux-gnuabi64",
 553             "mips64el-unknown-linux-gnuabi64",
 554             "mipsel-unknown-linux-gnu",
 555             "powerpc-unknown-linux-gnu",
 556             "powerpc64-unknown-linux-gnu",
 557             "powerpc64le-unknown-linux-gnu",
 558             "riscv64gc-unknown-linux-gnu",
 559             "s390x-unknown-linux-gnu",
 560             "x86_64-unknown-freebsd",
 561             "x86_64-unknown-illumos",
 562             "x86_64-unknown-linux-musl",
 563             "x86_64-unknown-netbsd",
 564         ]
 565         return opt == "true" \
 566             or (opt == "if-available" and self.build in supported_platforms)
 567
 568     def _download_component_helper(
 569         self, filename, pattern, tarball_suffix, stage0=True, key=None
 570     ):
 571         if key is None:
 572             if stage0:
 573                 key = self.stage0_compiler.date
 574             else:
 575                 key = self.rustc_commit
 576         cache_dst = os.path.join(self.build_dir, "cache")
 577         rustc_cache = os.path.join(cache_dst, key)
 578         if not os.path.exists(rustc_cache):
 579             os.makedirs(rustc_cache)
 580
 581         if stage0:
 582             base = self._download_url
 583             url = "dist/{}".format(key)
 584         else:
 585             base = "https://ci-artifacts.rust-lang.org"
 586             url = "rustc-builds/{}".format(self.rustc_commit)
 587         tarball = os.path.join(rustc_cache, filename)
 588         if not os.path.exists(tarball):
 589             get(
 590                 base,
 591                 "{}/{}".format(url, filename),
 592                 tarball,
 593                 self.checksums_sha256,
 594                 verbose=self.verbose,
 595                 do_verify=stage0,
 596             )
 597         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 598
 599     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 600         if not llvm_sha:
 601             print("error: could not find commit hash for downloading LLVM")
 602             print("help: maybe your repository history is too shallow?")
 603             print("help: consider disabling `download-ci-llvm`")
 604             print("help: or fetch enough history to include one upstream commit")
 605             exit(1)
 606         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 607         cache_dst = os.path.join(self.build_dir, "cache")
 608         rustc_cache = os.path.join(cache_dst, cache_prefix)
 609         if not os.path.exists(rustc_cache):
 610             os.makedirs(rustc_cache)
 611
 612         base = "https://ci-artifacts.rust-lang.org"
 613         url = "rustc-builds/{}".format(llvm_sha)
 614         if llvm_assertions:
 615             url = url.replace('rustc-builds', 'rustc-builds-alt')
 616         # ci-artifacts are only stored as .xz, not .gz
 617         if not support_xz():
 618             print("error: XZ support is required to download LLVM")
 619             print("help: consider disabling `download-ci-llvm` or using python3")
 620             exit(1)
 621         tarball_suffix = '.tar.xz'
 622         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 623         tarball = os.path.join(rustc_cache, filename)
 624         if not os.path.exists(tarball):
 625             get(
 626                 base,
 627                 "{}/{}".format(url, filename),
 628                 tarball,
 629                 self.checksums_sha256,
 630                 verbose=self.verbose,
 631                 do_verify=False,
 632             )
 633         unpack(tarball, tarball_suffix, self.llvm_root(),
 634                 match="rust-dev",
 635                 verbose=self.verbose)
 636
 637     def fix_bin_or_dylib(self, fname):
 638         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 639         or the RPATH section, to fix the dynamic library search path
 640
 641         This method is only required on NixOS and uses the PatchELF utility to
 642         change the interpreter/RPATH of ELF executables.
 643
 644         Please see https://nixos.org/patchelf.html for more information
 645         """
 646         default_encoding = sys.getdefaultencoding()
 647         try:
 648             ostype = subprocess.check_output(
 649                 ['uname', '-s']).strip().decode(default_encoding)
 650         except subprocess.CalledProcessError:
 651             return
 652         except OSError as reason:
 653             if getattr(reason, 'winerror', None) is not None:
 654                 return
 655             raise reason
 656
 657         if ostype != "Linux":
 658             return
 659
 660         # If the user has asked binaries to be patched for Nix, then
 661         # don't check for NixOS or `/lib`, just continue to the patching.
 662         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 663             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 664             # The latter one does not exist on NixOS when using tmpfs as root.
 665             try:
 666                 with open("/etc/os-release", "r") as f:
 667                     if not any(line.strip() == "ID=nixos" for line in f):
 668                         return
 669             except FileNotFoundError:
 670                 return
 671             if os.path.exists("/lib"):
 672                 return
 673
 674         # At this point we're pretty sure the user is running NixOS or
 675         # using Nix
 676         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 677         print(nix_os_msg, fname)
 678
 679         # Only build `.nix-deps` once.
 680         nix_deps_dir = self.nix_deps_dir
 681         if not nix_deps_dir:
 682             # Run `nix-build` to "build" each dependency (which will likely reuse
 683             # the existing `/nix/store` copy, or at most download a pre-built copy).
 684             #
 685             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 686             # directory, but still reference the actual `/nix/store` path in the rpath
 687             # as it makes it significantly more robust against changes to the location of
 688             # the `.nix-deps` location.
 689             #
 690             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 691             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 692             # patchelf: Needed for patching ELF binaries (see doc comment above).
 693             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 694             nix_expr = '''
 695             with (import <nixpkgs> {});
 696             symlinkJoin {
 697               name = "rust-stage0-dependencies";
 698               paths = [
 699                 zlib
 700                 patchelf
 701                 stdenv.cc.bintools
 702               ];
 703             }
 704             '''
 705             try:
 706                 subprocess.check_output([
 707                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 708                 ])
 709             except subprocess.CalledProcessError as reason:
 710                 print("warning: failed to call nix-build:", reason)
 711                 return
 712             self.nix_deps_dir = nix_deps_dir
 713
 714         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 715         rpath_entries = [
 716             # Relative default, all binary and dynamic libraries we ship
 717             # appear to have this (even when `../lib` is redundant).
 718             "$ORIGIN/../lib",
 719             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 720         ]
 721         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 722         if not fname.endswith(".so"):
 723             # Finally, set the corret .interp for binaries
 724             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 725                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 726
 727         try:
 728             subprocess.check_output([patchelf] + patchelf_args + [fname])
 729         except subprocess.CalledProcessError as reason:
 730             print("warning: failed to call patchelf:", reason)
 731             return
 732
 733     # If `download-rustc` is set, download the most recent commit with CI artifacts
 734     def maybe_download_ci_toolchain(self):
 735         # If `download-rustc` is not set, default to rebuilding.
 736         download_rustc = self.get_toml("download-rustc", section="rust")
 737         if download_rustc is None or download_rustc == "false":
 738             return None
 739         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 740
 741         # Handle running from a directory other than the top level
 742         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 743         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 744         compiler = "{}/compiler/".format(top_level)
 745         library = "{}/library/".format(top_level)
 746
 747         # Look for a version to compare to based on the current commit.
 748         # Only commits merged by bors will have CI artifacts.
 749         merge_base = [
 750             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 751             "--first-parent", "HEAD"
 752         ]
 753         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 754         if not commit:
 755             print("error: could not find commit hash for downloading rustc")
 756             print("help: maybe your repository history is too shallow?")
 757             print("help: consider disabling `download-rustc`")
 758             print("help: or fetch enough history to include one upstream commit")
 759             exit(1)
 760
 761         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 762         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 763         if status != 0:
 764             if download_rustc == "if-unchanged":
 765                 if self.verbose:
 766                     print("warning: saw changes to compiler/ or library/ since {}; " \
 767                           "ignoring `download-rustc`".format(commit))
 768                 return None
 769             print("warning: `download-rustc` is enabled, but there are changes to " \
 770                   "compiler/ or library/")
 771
 772         if self.verbose:
 773             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
 774         self.rustc_commit = commit
 775         # FIXME: support downloading artifacts from the beta channel
 776         self.download_toolchain(False, "nightly")
 777
 778     def rustc_stamp(self, stage0):
 779         """Return the path for .rustc-stamp at the given stage
 780
 781         >>> rb = RustBuild()
 782         >>> rb.build_dir = "build"
 783         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 784         True
 785         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 786         True
 787         """
 788         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 789
 790     def rustfmt_stamp(self):
 791         """Return the path for .rustfmt-stamp
 792
 793         >>> rb = RustBuild()
 794         >>> rb.build_dir = "build"
 795         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 796         True
 797         """
 798         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 799
 800     def llvm_stamp(self):
 801         """Return the path for .rustfmt-stamp
 802
 803         >>> rb = RustBuild()
 804         >>> rb.build_dir = "build"
 805         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 806         True
 807         """
 808         return os.path.join(self.llvm_root(), '.llvm-stamp')
 809
 810
 811     def program_out_of_date(self, stamp_path, key):
 812         """Check if the given program stamp is out of date"""
 813         if not os.path.exists(stamp_path) or self.clean:
 814             return True
 815         with open(stamp_path, 'r') as stamp:
 816             return key != stamp.read()
 817
 818     def bin_root(self, stage0):
 819         """Return the binary root directory for the given stage
 820
 821         >>> rb = RustBuild()
 822         >>> rb.build_dir = "build"
 823         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 824         True
 825         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 826         True
 827
 828         When the 'build' property is given should be a nested directory:
 829
 830         >>> rb.build = "devel"
 831         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 832         True
 833         """
 834         if stage0:
 835             subdir = "stage0"
 836         else:
 837             subdir = "ci-rustc"
 838         return os.path.join(self.build_dir, self.build, subdir)
 839
 840     def llvm_root(self):
 841         """Return the CI LLVM root directory
 842
 843         >>> rb = RustBuild()
 844         >>> rb.build_dir = "build"
 845         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 846         True
 847
 848         When the 'build' property is given should be a nested directory:
 849
 850         >>> rb.build = "devel"
 851         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 852         True
 853         """
 854         return os.path.join(self.build_dir, self.build, "ci-llvm")
 855
 856     def get_toml(self, key, section=None):
 857         """Returns the value of the given key in config.toml, otherwise returns None
 858
 859         >>> rb = RustBuild()
 860         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 861         >>> rb.get_toml("key2")
 862         'value2'
 863
 864         If the key does not exists, the result is None:
 865
 866         >>> rb.get_toml("key3") is None
 867         True
 868
 869         Optionally also matches the section the key appears in
 870
 871         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 872         >>> rb.get_toml('key', 'a')
 873         'value1'
 874         >>> rb.get_toml('key', 'b')
 875         'value2'
 876         >>> rb.get_toml('key', 'c') is None
 877         True
 878
 879         >>> rb.config_toml = 'key1 = true'
 880         >>> rb.get_toml("key1")
 881         'true'
 882         """
 883
 884         cur_section = None
 885         for line in self.config_toml.splitlines():
 886             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 887             if section_match is not None:
 888                 cur_section = section_match.group(1)
 889
 890             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 891             if match is not None:
 892                 value = match.group(1)
 893                 if section is None or section == cur_section:
 894                     return self.get_string(value) or value.strip()
 895         return None
 896
 897     def cargo(self):
 898         """Return config path for cargo"""
 899         return self.program_config('cargo')
 900
 901     def rustc(self, stage0):
 902         """Return config path for rustc"""
 903         return self.program_config('rustc', stage0)
 904
 905     def rustfmt(self):
 906         """Return config path for rustfmt"""
 907         if self.stage0_rustfmt is None:
 908             return None
 909         return self.program_config('rustfmt')
 910
 911     def program_config(self, program, stage0=True):
 912         """Return config path for the given program at the given stage
 913
 914         >>> rb = RustBuild()
 915         >>> rb.config_toml = 'rustc = "rustc"\\n'
 916         >>> rb.program_config('rustc')
 917         'rustc'
 918         >>> rb.config_toml = ''
 919         >>> cargo_path = rb.program_config('cargo', True)
 920         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 921         ... "bin", "cargo")
 922         True
 923         >>> cargo_path = rb.program_config('cargo', False)
 924         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 925         ... "bin", "cargo")
 926         True
 927         """
 928         config = self.get_toml(program)
 929         if config:
 930             return os.path.expanduser(config)
 931         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 932             program, self.exe_suffix()))
 933
 934     @staticmethod
 935     def get_string(line):
 936         """Return the value between double quotes
 937
 938         >>> RustBuild.get_string('    "devel"   ')
 939         'devel'
 940         >>> RustBuild.get_string("    'devel'   ")
 941         'devel'
 942         >>> RustBuild.get_string('devel') is None
 943         True
 944         >>> RustBuild.get_string('    "devel   ')
 945         ''
 946         """
 947         start = line.find('"')
 948         if start != -1:
 949             end = start + 1 + line[start + 1:].find('"')
 950             return line[start + 1:end]
 951         start = line.find('\'')
 952         if start != -1:
 953             end = start + 1 + line[start + 1:].find('\'')
 954             return line[start + 1:end]
 955         return None
 956
 957     @staticmethod
 958     def exe_suffix():
 959         """Return a suffix for executables"""
 960         if sys.platform == 'win32':
 961             return '.exe'
 962         return ''
 963
 964     def bootstrap_binary(self):
 965         """Return the path of the bootstrap binary
 966
 967         >>> rb = RustBuild()
 968         >>> rb.build_dir = "build"
 969         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 970         ... "debug", "bootstrap")
 971         True
 972         """
 973         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 974
 975     def build_bootstrap(self):
 976         """Build bootstrap"""
 977         build_dir = os.path.join(self.build_dir, "bootstrap")
 978         if self.clean and os.path.exists(build_dir):
 979             shutil.rmtree(build_dir)
 980         env = os.environ.copy()
 981         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 982         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 983         if "CARGO_BUILD_TARGET" in env:
 984             del env["CARGO_BUILD_TARGET"]
 985         env["CARGO_TARGET_DIR"] = build_dir
 986         env["RUSTC"] = self.rustc(True)
 987         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 988             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 989             if "LD_LIBRARY_PATH" in env else ""
 990         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 991             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 992             if "DYLD_LIBRARY_PATH" in env else ""
 993         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 994             (os.pathsep + env["LIBRARY_PATH"]) \
 995             if "LIBRARY_PATH" in env else ""
 996
 997         # preserve existing RUSTFLAGS
 998         env.setdefault("RUSTFLAGS", "")
 999         build_section = "target.{}".format(self.build)
1000         target_features = []
1001         if self.get_toml("crt-static", build_section) == "true":
1002             target_features += ["+crt-static"]
1003         elif self.get_toml("crt-static", build_section) == "false":
1004             target_features += ["-crt-static"]
1005         if target_features:
1006             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1007         target_linker = self.get_toml("linker", build_section)
1008         if target_linker is not None:
1009             env["RUSTFLAGS"] += " -C linker=" + target_linker
1010         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1011         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1012         if self.get_toml("deny-warnings", "rust") != "false":
1013             env["RUSTFLAGS"] += " -Dwarnings"
1014
1015         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1016             os.pathsep + env["PATH"]
1017         if not os.path.isfile(self.cargo()):
1018             raise Exception("no cargo executable found at `{}`".format(
1019                 self.cargo()))
1020         args = [self.cargo(), "build", "--manifest-path",
1021                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1022         for _ in range(0, self.verbose):
1023             args.append("--verbose")
1024         if self.use_locked_deps:
1025             args.append("--locked")
1026         if self.use_vendored_sources:
1027             args.append("--frozen")
1028         run(args, env=env, verbose=self.verbose)
1029
1030     def build_triple(self):
1031         """Build triple as in LLVM
1032
1033         Note that `default_build_triple` is moderately expensive,
1034         so use `self.build` where possible.
1035         """
1036         config = self.get_toml('build')
1037         if config:
1038             return config
1039         return default_build_triple(self.verbose)
1040
1041     def check_submodule(self, module, slow_submodules):
1042         if not slow_submodules:
1043             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1044                                            cwd=os.path.join(self.rust_root, module),
1045                                            stdout=subprocess.PIPE)
1046             return checked_out
1047         else:
1048             return None
1049
1050     def update_submodule(self, module, checked_out, recorded_submodules):
1051         module_path = os.path.join(self.rust_root, module)
1052
1053         if checked_out is not None:
1054             default_encoding = sys.getdefaultencoding()
1055             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1056             if recorded_submodules[module] == checked_out:
1057                 return
1058
1059         print("Updating submodule", module)
1060
1061         run(["git", "submodule", "-q", "sync", module],
1062             cwd=self.rust_root, verbose=self.verbose)
1063
1064         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1065         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1066             update_args.append("--progress")
1067         update_args.append(module)
1068         try:
1069             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1070         except RuntimeError:
1071             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1072             print('Either stash the changes by running "git stash" within the submodule\'s')
1073             print('directory, reset them by running "git reset --hard", or commit them.')
1074             print("To reset all submodules' changes run", end=" ")
1075             print('"git submodule foreach --recursive git reset --hard".')
1076             raise SystemExit(1)
1077
1078         run(["git", "reset", "-q", "--hard"],
1079             cwd=module_path, verbose=self.verbose)
1080         run(["git", "clean", "-qdfx"],
1081             cwd=module_path, verbose=self.verbose)
1082
1083     def update_submodules(self):
1084         """Update submodules"""
1085         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1086                 self.get_toml('submodules') == "false":
1087             return
1088
1089         default_encoding = sys.getdefaultencoding()
1090
1091         # check the existence and version of 'git' command
1092         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1093         self.git_version = distutils.version.LooseVersion(git_version_str)
1094
1095         slow_submodules = self.get_toml('fast-submodules') == "false"
1096         start_time = time()
1097         if slow_submodules:
1098             print('Unconditionally updating submodules')
1099         else:
1100             print('Updating only changed submodules')
1101         default_encoding = sys.getdefaultencoding()
1102         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1103         # currently requires everything in a workspace to be "locally present" when starting a
1104         # build, and will give a hard error if any Cargo.toml files are missing.
1105         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1106         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1107         #   That will still require cloning the submodules the second you check the standard
1108         #   library, though...
1109         # FIXME: Is there a way to avoid hard-coding the submodules required?
1110         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1111         submodules = [
1112             "src/tools/rust-installer",
1113             "src/tools/cargo",
1114             "src/tools/rls",
1115             "src/tools/miri",
1116             "library/backtrace",
1117             "library/stdarch"
1118         ]
1119         filtered_submodules = []
1120         submodules_names = []
1121         for module in submodules:
1122             check = self.check_submodule(module, slow_submodules)
1123             filtered_submodules.append((module, check))
1124             submodules_names.append(module)
1125         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1126                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1127         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1128         # { filename: hash }
1129         recorded_submodules = {}
1130         for data in recorded:
1131             # [mode, kind, hash, filename]
1132             data = data.split()
1133             recorded_submodules[data[3]] = data[2]
1134         for module in filtered_submodules:
1135             self.update_submodule(module[0], module[1], recorded_submodules)
1136         print("Submodules updated in %.2f seconds" % (time() - start_time))
1137
1138     def set_dist_environment(self, url):
1139         """Set download URL for normal environment"""
1140         if 'RUSTUP_DIST_SERVER' in os.environ:
1141             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1142         else:
1143             self._download_url = url
1144
1145     def check_vendored_status(self):
1146         """Check that vendoring is configured properly"""
1147         vendor_dir = os.path.join(self.rust_root, 'vendor')
1148         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1149             if os.environ.get('USER') != os.environ['SUDO_USER']:
1150                 self.use_vendored_sources = True
1151                 print('info: looks like you are running this command under `sudo`')
1152                 print('      and so in order to preserve your $HOME this will now')
1153                 print('      use vendored sources by default.')
1154                 if not os.path.exists(vendor_dir):
1155                     print('error: vendoring required, but vendor directory does not exist.')
1156                     print('       Run `cargo vendor` without sudo to initialize the '
1157                           'vendor directory.')
1158                     raise Exception("{} not found".format(vendor_dir))
1159
1160         if self.use_vendored_sources:
1161             config = ("[source.crates-io]\n"
1162                       "replace-with = 'vendored-sources'\n"
1163                       "registry = 'https://example.com'\n"
1164                       "\n"
1165                       "[source.vendored-sources]\n"
1166                       "directory = '{}/vendor'\n"
1167                       .format(self.rust_root))
1168             if not os.path.exists('.cargo'):
1169                 os.makedirs('.cargo')
1170                 with output('.cargo/config') as cargo_config:
1171                     cargo_config.write(config)
1172             else:
1173                 print('info: using vendored source, but .cargo/config is already present.')
1174                 print('      Reusing the current configuration file. But you may want to '
1175                       'configure vendoring like this:')
1176                 print(config)
1177         else:
1178             if os.path.exists('.cargo'):
1179                 shutil.rmtree('.cargo')
1180
1181     def ensure_vendored(self):
1182         """Ensure that the vendored sources are available if needed"""
1183         vendor_dir = os.path.join(self.rust_root, 'vendor')
1184         # Note that this does not handle updating the vendored dependencies if
1185         # the rust git repository is updated. Normal development usually does
1186         # not use vendoring, so hopefully this isn't too much of a problem.
1187         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1188             run([
1189                 self.cargo(),
1190                 "vendor",
1191                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1192                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1193             ], verbose=self.verbose, cwd=self.rust_root)
1194
1195
1196 def bootstrap(help_triggered):
1197     """Configure, fetch, build and run the initial bootstrap"""
1198
1199     # If the user is asking for help, let them know that the whole download-and-build
1200     # process has to happen before anything is printed out.
1201     if help_triggered:
1202         print("info: Downloading and building bootstrap before processing --help")
1203         print("      command. See src/bootstrap/README.md for help with common")
1204         print("      commands.")
1205
1206     parser = argparse.ArgumentParser(description='Build rust')
1207     parser.add_argument('--config')
1208     parser.add_argument('--build')
1209     parser.add_argument('--clean', action='store_true')
1210     parser.add_argument('-v', '--verbose', action='count', default=0)
1211
1212     args = [a for a in sys.argv if a != '-h' and a != '--help']
1213     args, _ = parser.parse_known_args(args)
1214
1215     # Configure initial bootstrap
1216     build = RustBuild()
1217     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1218     build.verbose = args.verbose
1219     build.clean = args.clean
1220
1221     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1222     # exists).
1223     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1224     if not toml_path and os.path.exists('config.toml'):
1225         toml_path = 'config.toml'
1226
1227     if toml_path:
1228         if not os.path.exists(toml_path):
1229             toml_path = os.path.join(build.rust_root, toml_path)
1230
1231         with open(toml_path) as config:
1232             build.config_toml = config.read()
1233
1234     profile = build.get_toml('profile')
1235     if profile is not None:
1236         include_file = 'config.{}.toml'.format(profile)
1237         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1238         include_path = os.path.join(include_dir, include_file)
1239         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1240         # specific key, so appending our defaults at the end allows the user to override them
1241         with open(include_path) as included_toml:
1242             build.config_toml += os.linesep + included_toml.read()
1243
1244     config_verbose = build.get_toml('verbose', 'build')
1245     if config_verbose is not None:
1246         build.verbose = max(build.verbose, int(config_verbose))
1247
1248     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1249
1250     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1251
1252     build.check_vendored_status()
1253
1254     build_dir = build.get_toml('build-dir', 'build') or 'build'
1255     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1256
1257     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1258         data = json.load(f)
1259     build.checksums_sha256 = data["checksums_sha256"]
1260     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1261     if data.get("rustfmt") is not None:
1262         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1263
1264     build.set_dist_environment(data["dist_server"])
1265
1266     build.build = args.build or build.build_triple()
1267
1268     # Acquire the lock before doing any build actions
1269     # The lock is released when `lock` is dropped
1270     if not os.path.exists(build.build_dir):
1271         os.makedirs(build.build_dir)
1272     lock = acquire_lock(build.build_dir)
1273     build.update_submodules()
1274
1275     # Fetch/build the bootstrap
1276     build.download_toolchain()
1277     # Download the master compiler if `download-rustc` is set
1278     build.maybe_download_ci_toolchain()
1279     sys.stdout.flush()
1280     build.ensure_vendored()
1281     build.build_bootstrap()
1282     sys.stdout.flush()
1283
1284     # Run the bootstrap
1285     args = [build.bootstrap_binary()]
1286     args.extend(sys.argv[1:])
1287     env = os.environ.copy()
1288     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1289     env["BOOTSTRAP_PYTHON"] = sys.executable
1290     env["BUILD_DIR"] = build.build_dir
1291     env["RUSTC_BOOTSTRAP"] = '1'
1292     if toml_path:
1293         env["BOOTSTRAP_CONFIG"] = toml_path
1294     if build.rustc_commit is not None:
1295         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1296     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1297
1298
1299 def main():
1300     """Entry point for the bootstrap process"""
1301     start_time = time()
1302
1303     # x.py help <cmd> ...
1304     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1305         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1306
1307     help_triggered = (
1308         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1309     try:
1310         bootstrap(help_triggered)
1311         if not help_triggered:
1312             print("Build completed successfully in {}".format(
1313                 format_build_time(time() - start_time)))
1314     except (SystemExit, KeyboardInterrupt) as error:
1315         if hasattr(error, 'code') and isinstance(error.code, int):
1316             exit_code = error.code
1317         else:
1318             exit_code = 1
1319             print(error)
1320         if not help_triggered:
1321             print("Build completed unsuccessfully in {}".format(
1322                 format_build_time(time() - start_time)))
1323         sys.exit(exit_code)
1324
1325
1326 if __name__ == '__main__':
1327     main()