src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time, sleep
  17
  18 # Acquire a lock on the build directory to make sure that
  19 # we don't cause a race condition while building
  20 # Lock is created in `build_dir/lock.db`
  21 def acquire_lock(build_dir):
  22     try:
  23         import sqlite3
  24
  25         path = os.path.join(build_dir, "lock.db")
  26         try:
  27             con = sqlite3.Connection(path, timeout=0)
  28             curs = con.cursor()
  29             curs.execute("BEGIN EXCLUSIVE")
  30             # The lock is released when the cursor is dropped
  31             return curs
  32         # If the database is busy then lock has already been acquired
  33         # so we wait for the lock.
  34         # We retry every quarter second so that execution is passed back to python
  35         # so that it can handle signals
  36         except sqlite3.OperationalError:
  37             del con
  38             del curs
  39             print("Waiting for lock on build directory")
  40             con = sqlite3.Connection(path, timeout=0.25)
  41             curs = con.cursor()
  42             while True:
  43                 try:
  44                     curs.execute("BEGIN EXCLUSIVE")
  45                     break
  46                 except sqlite3.OperationalError:
  47                     pass
  48                 sleep(0.25)
  49             return curs
  50     except ImportError:
  51         print("warning: sqlite3 not available in python, skipping build directory lock")
  52         print("please file an issue on rust-lang/rust")
  53         print("this is not a problem for non-concurrent x.py invocations")
  54         return None
  55
  56 def support_xz():
  57     try:
  58         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  59             temp_path = temp_file.name
  60         with tarfile.open(temp_path, "w:xz"):
  61             pass
  62         return True
  63     except tarfile.CompressionError:
  64         return False
  65
  66 def get(base, url, path, checksums, verbose=False, do_verify=True, help_on_error=None):
  67     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  68         temp_path = temp_file.name
  69
  70     try:
  71         if do_verify:
  72             if url not in checksums:
  73                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  74             sha256 = checksums[url]
  75             if os.path.exists(path):
  76                 if verify(path, sha256, False):
  77                     if verbose:
  78                         print("using already-download file", path)
  79                     return
  80                 else:
  81                     if verbose:
  82                         print("ignoring already-download file",
  83                             path, "due to failed verification")
  84                     os.unlink(path)
  85         download(temp_path, "{}/{}".format(base, url), True, verbose, help_on_error=help_on_error)
  86         if do_verify and not verify(temp_path, sha256, verbose):
  87             raise RuntimeError("failed verification")
  88         if verbose:
  89             print("moving {} to {}".format(temp_path, path))
  90         shutil.move(temp_path, path)
  91     finally:
  92         if os.path.isfile(temp_path):
  93             if verbose:
  94                 print("removing", temp_path)
  95             os.unlink(temp_path)
  96
  97
  98 def download(path, url, probably_big, verbose, help_on_error=None):
  99     for _ in range(0, 4):
 100         try:
 101             _download(path, url, probably_big, verbose, True, help_on_error=help_on_error)
 102             return
 103         except RuntimeError:
 104             print("\nspurious failure, trying again")
 105     _download(path, url, probably_big, verbose, False, help_on_error=help_on_error)
 106
 107
 108 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
 109     if probably_big or verbose:
 110         print("downloading {}".format(url))
 111     # see https://serverfault.com/questions/301128/how-to-download
 112     if sys.platform == 'win32':
 113         run(["PowerShell.exe", "/nologo", "-Command",
 114              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 115              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
 116             verbose=verbose,
 117             exception=exception)
 118     else:
 119         if probably_big or verbose:
 120             option = "-#"
 121         else:
 122             option = "-s"
 123         require(["curl", "--version"])
 124         run(["curl", option,
 125              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
 126              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
 127              "--retry", "3", "-Sf", "-o", path, url],
 128             verbose=verbose,
 129             exception=exception,
 130             help_on_error=help_on_error)
 131
 132
 133 def verify(path, expected, verbose):
 134     """Check if the sha256 sum of the given path is valid"""
 135     if verbose:
 136         print("verifying", path)
 137     with open(path, "rb") as source:
 138         found = hashlib.sha256(source.read()).hexdigest()
 139     verified = found == expected
 140     if not verified:
 141         print("invalid checksum:\n"
 142               "    found:    {}\n"
 143               "    expected: {}".format(found, expected))
 144     return verified
 145
 146
 147 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 148     """Unpack the given tarball file"""
 149     print("extracting", tarball)
 150     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 151     with contextlib.closing(tarfile.open(tarball)) as tar:
 152         for member in tar.getnames():
 153             if "/" not in member:
 154                 continue
 155             name = member.replace(fname + "/", "", 1)
 156             if match is not None and not name.startswith(match):
 157                 continue
 158             name = name[len(match) + 1:]
 159
 160             dst_path = os.path.join(dst, name)
 161             if verbose:
 162                 print("  extracting", member)
 163             tar.extract(member, dst)
 164             src_path = os.path.join(dst, member)
 165             if os.path.isdir(src_path) and os.path.exists(dst_path):
 166                 continue
 167             shutil.move(src_path, dst_path)
 168     shutil.rmtree(os.path.join(dst, fname))
 169
 170
 171 def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=None, **kwargs):
 172     """Run a child program in a new process"""
 173     if verbose:
 174         print("running: " + ' '.join(args))
 175     sys.stdout.flush()
 176     # Use Popen here instead of call() as it apparently allows powershell on
 177     # Windows to not lock up waiting for input presumably.
 178     ret = subprocess.Popen(args, **kwargs)
 179     code = ret.wait()
 180     if code != 0:
 181         err = "failed to run: " + ' '.join(args)
 182         if help_on_error is not None:
 183             err += "\n" + help_on_error
 184         if verbose or exception:
 185             raise RuntimeError(err)
 186         # For most failures, we definitely do want to print this error, or the user will have no
 187         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 188         # have already printed an error above, so there's no need to print the exact command we're
 189         # running.
 190         if is_bootstrap:
 191             sys.exit(1)
 192         else:
 193             sys.exit(err)
 194
 195
 196 def require(cmd, exit=True):
 197     '''Run a command, returning its output.
 198     On error,
 199         If `exit` is `True`, exit the process.
 200         Otherwise, return None.'''
 201     try:
 202         return subprocess.check_output(cmd).strip()
 203     except (subprocess.CalledProcessError, OSError) as exc:
 204         if not exit:
 205             return None
 206         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 207         print("Please make sure it's installed and in the path.")
 208         sys.exit(1)
 209
 210
 211 def format_build_time(duration):
 212     """Return a nicer format for build time
 213
 214     >>> format_build_time('300')
 215     '0:05:00'
 216     """
 217     return str(datetime.timedelta(seconds=int(duration)))
 218
 219
 220 def default_build_triple(verbose):
 221     """Build triple as in LLVM"""
 222     # If the user already has a host build triple with an existing `rustc`
 223     # install, use their preference. This fixes most issues with Windows builds
 224     # being detected as GNU instead of MSVC.
 225     default_encoding = sys.getdefaultencoding()
 226     try:
 227         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 228                 stderr=subprocess.DEVNULL)
 229         version = version.decode(default_encoding)
 230         host = next(x for x in version.split('\n') if x.startswith("host: "))
 231         triple = host.split("host: ")[1]
 232         if verbose:
 233             print("detected default triple {} from pre-installed rustc".format(triple))
 234         return triple
 235     except Exception as e:
 236         if verbose:
 237             print("pre-installed rustc not detected: {}".format(e))
 238             print("falling back to auto-detect")
 239
 240     required = sys.platform != 'win32'
 241     ostype = require(["uname", "-s"], exit=required)
 242     cputype = require(['uname', '-m'], exit=required)
 243
 244     # If we do not have `uname`, assume Windows.
 245     if ostype is None or cputype is None:
 246         return 'x86_64-pc-windows-msvc'
 247
 248     ostype = ostype.decode(default_encoding)
 249     cputype = cputype.decode(default_encoding)
 250
 251     # The goal here is to come up with the same triple as LLVM would,
 252     # at least for the subset of platforms we're willing to target.
 253     ostype_mapper = {
 254         'Darwin': 'apple-darwin',
 255         'DragonFly': 'unknown-dragonfly',
 256         'FreeBSD': 'unknown-freebsd',
 257         'Haiku': 'unknown-haiku',
 258         'NetBSD': 'unknown-netbsd',
 259         'OpenBSD': 'unknown-openbsd'
 260     }
 261
 262     # Consider the direct transformation first and then the special cases
 263     if ostype in ostype_mapper:
 264         ostype = ostype_mapper[ostype]
 265     elif ostype == 'Linux':
 266         os_from_sp = subprocess.check_output(
 267             ['uname', '-o']).strip().decode(default_encoding)
 268         if os_from_sp == 'Android':
 269             ostype = 'linux-android'
 270         else:
 271             ostype = 'unknown-linux-gnu'
 272     elif ostype == 'SunOS':
 273         ostype = 'pc-solaris'
 274         # On Solaris, uname -m will return a machine classification instead
 275         # of a cpu type, so uname -p is recommended instead.  However, the
 276         # output from that option is too generic for our purposes (it will
 277         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 278         # must be used instead.
 279         cputype = require(['isainfo', '-k']).decode(default_encoding)
 280         # sparc cpus have sun as a target vendor
 281         if 'sparc' in cputype:
 282             ostype = 'sun-solaris'
 283     elif ostype.startswith('MINGW'):
 284         # msys' `uname` does not print gcc configuration, but prints msys
 285         # configuration. so we cannot believe `uname -m`:
 286         # msys1 is always i686 and msys2 is always x86_64.
 287         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 288         # MINGW64 on x86_64.
 289         ostype = 'pc-windows-gnu'
 290         cputype = 'i686'
 291         if os.environ.get('MSYSTEM') == 'MINGW64':
 292             cputype = 'x86_64'
 293     elif ostype.startswith('MSYS'):
 294         ostype = 'pc-windows-gnu'
 295     elif ostype.startswith('CYGWIN_NT'):
 296         cputype = 'i686'
 297         if ostype.endswith('WOW64'):
 298             cputype = 'x86_64'
 299         ostype = 'pc-windows-gnu'
 300     elif sys.platform == 'win32':
 301         # Some Windows platforms might have a `uname` command that returns a
 302         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 303         # these cases, fall back to using sys.platform.
 304         return 'x86_64-pc-windows-msvc'
 305     else:
 306         err = "unknown OS type: {}".format(ostype)
 307         sys.exit(err)
 308
 309     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
 310         cputype = subprocess.check_output(
 311               ['uname', '-p']).strip().decode(default_encoding)
 312     cputype_mapper = {
 313         'BePC': 'i686',
 314         'aarch64': 'aarch64',
 315         'amd64': 'x86_64',
 316         'arm64': 'aarch64',
 317         'i386': 'i686',
 318         'i486': 'i686',
 319         'i686': 'i686',
 320         'i786': 'i686',
 321         'm68k': 'm68k',
 322         'powerpc': 'powerpc',
 323         'powerpc64': 'powerpc64',
 324         'powerpc64le': 'powerpc64le',
 325         'ppc': 'powerpc',
 326         'ppc64': 'powerpc64',
 327         'ppc64le': 'powerpc64le',
 328         'riscv64': 'riscv64gc',
 329         's390x': 's390x',
 330         'x64': 'x86_64',
 331         'x86': 'i686',
 332         'x86-64': 'x86_64',
 333         'x86_64': 'x86_64'
 334     }
 335
 336     # Consider the direct transformation first and then the special cases
 337     if cputype in cputype_mapper:
 338         cputype = cputype_mapper[cputype]
 339     elif cputype in {'xscale', 'arm'}:
 340         cputype = 'arm'
 341         if ostype == 'linux-android':
 342             ostype = 'linux-androideabi'
 343         elif ostype == 'unknown-freebsd':
 344             cputype = subprocess.check_output(
 345                 ['uname', '-p']).strip().decode(default_encoding)
 346             ostype = 'unknown-freebsd'
 347     elif cputype == 'armv6l':
 348         cputype = 'arm'
 349         if ostype == 'linux-android':
 350             ostype = 'linux-androideabi'
 351         else:
 352             ostype += 'eabihf'
 353     elif cputype in {'armv7l', 'armv8l'}:
 354         cputype = 'armv7'
 355         if ostype == 'linux-android':
 356             ostype = 'linux-androideabi'
 357         else:
 358             ostype += 'eabihf'
 359     elif cputype == 'mips':
 360         if sys.byteorder == 'big':
 361             cputype = 'mips'
 362         elif sys.byteorder == 'little':
 363             cputype = 'mipsel'
 364         else:
 365             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 366     elif cputype == 'mips64':
 367         if sys.byteorder == 'big':
 368             cputype = 'mips64'
 369         elif sys.byteorder == 'little':
 370             cputype = 'mips64el'
 371         else:
 372             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 373         # only the n64 ABI is supported, indicate it
 374         ostype += 'abi64'
 375     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 376         pass
 377     else:
 378         err = "unknown cpu type: {}".format(cputype)
 379         sys.exit(err)
 380
 381     return "{}-{}".format(cputype, ostype)
 382
 383
 384 @contextlib.contextmanager
 385 def output(filepath):
 386     tmp = filepath + '.tmp'
 387     with open(tmp, 'w') as f:
 388         yield f
 389     try:
 390         if os.path.exists(filepath):
 391             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 392     except OSError:
 393         shutil.copy2(tmp, filepath)
 394         os.remove(tmp)
 395         return
 396     os.rename(tmp, filepath)
 397
 398
 399 class Stage0Toolchain:
 400     def __init__(self, stage0_payload):
 401         self.date = stage0_payload["date"]
 402         self.version = stage0_payload["version"]
 403
 404     def channel(self):
 405         return self.version + "-" + self.date
 406
 407
 408 class RustBuild(object):
 409     """Provide all the methods required to build Rust"""
 410     def __init__(self):
 411         self.checksums_sha256 = {}
 412         self.stage0_compiler = None
 413         self.stage0_rustfmt = None
 414         self._download_url = ''
 415         self.build = ''
 416         self.build_dir = ''
 417         self.clean = False
 418         self.config_toml = ''
 419         self.rust_root = ''
 420         self.use_locked_deps = ''
 421         self.use_vendored_sources = ''
 422         self.verbose = False
 423         self.git_version = None
 424         self.nix_deps_dir = None
 425         self.rustc_commit = None
 426
 427     def download_toolchain(self, stage0=True, rustc_channel=None):
 428         """Fetch the build system for Rust, written in Rust
 429
 430         This method will build a cache directory, then it will fetch the
 431         tarball which has the stage0 compiler used to then bootstrap the Rust
 432         compiler itself.
 433
 434         Each downloaded tarball is extracted, after that, the script
 435         will move all the content to the right place.
 436         """
 437         if rustc_channel is None:
 438             rustc_channel = self.stage0_compiler.version
 439         bin_root = self.bin_root(stage0)
 440
 441         key = self.stage0_compiler.date
 442         if not stage0:
 443             key += str(self.rustc_commit)
 444         if self.rustc(stage0).startswith(bin_root) and \
 445                 (not os.path.exists(self.rustc(stage0)) or
 446                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 447             if os.path.exists(bin_root):
 448                 shutil.rmtree(bin_root)
 449             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 450             filename = "rust-std-{}-{}{}".format(
 451                 rustc_channel, self.build, tarball_suffix)
 452             pattern = "rust-std-{}".format(self.build)
 453             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 454             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 455                                               tarball_suffix)
 456             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 457             # download-rustc doesn't need its own cargo, it can just use beta's.
 458             if stage0:
 459                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 460                                                 tarball_suffix)
 461                 self._download_component_helper(filename, "cargo", tarball_suffix)
 462                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 463             else:
 464                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 465                 self._download_component_helper(
 466                     filename, "rustc-dev", tarball_suffix, stage0
 467                 )
 468
 469             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 470             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 471             lib_dir = "{}/lib".format(bin_root)
 472             for lib in os.listdir(lib_dir):
 473                 if lib.endswith(".so"):
 474                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 475             with output(self.rustc_stamp(stage0)) as rust_stamp:
 476                 rust_stamp.write(key)
 477
 478         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 479             not os.path.exists(self.rustfmt())
 480             or self.program_out_of_date(
 481                 self.rustfmt_stamp(),
 482                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 483             )
 484         ):
 485             if self.stage0_rustfmt is not None:
 486                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 487                 filename = "rustfmt-{}-{}{}".format(
 488                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 489                 )
 490                 self._download_component_helper(
 491                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 492                 )
 493                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 494                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 495                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 496                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 497
 498         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 499         if self.downloading_llvm() and stage0:
 500             # We want the most recent LLVM submodule update to avoid downloading
 501             # LLVM more often than necessary.
 502             #
 503             # This git command finds that commit SHA, looking for bors-authored
 504             # commits that modified src/llvm-project or other relevant version
 505             # stamp files.
 506             #
 507             # This works even in a repository that has not yet initialized
 508             # submodules.
 509             top_level = subprocess.check_output([
 510                 "git", "rev-parse", "--show-toplevel",
 511             ]).decode(sys.getdefaultencoding()).strip()
 512             llvm_sha = subprocess.check_output([
 513                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 514                 "--first-parent", "HEAD",
 515                 "--",
 516                 "{}/src/llvm-project".format(top_level),
 517                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 518                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 519                 "{}/src/version".format(top_level)
 520             ]).decode(sys.getdefaultencoding()).strip()
 521             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 522             llvm_root = self.llvm_root()
 523             llvm_lib = os.path.join(llvm_root, "lib")
 524             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 525                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 526                 for binary in ["llvm-config", "FileCheck"]:
 527                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 528                 for lib in os.listdir(llvm_lib):
 529                     if lib.endswith(".so"):
 530                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 531                 with output(self.llvm_stamp()) as llvm_stamp:
 532                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 533
 534     def downloading_llvm(self):
 535         opt = self.get_toml('download-ci-llvm', 'llvm')
 536         # This is currently all tier 1 targets and tier 2 targets with host tools
 537         # (since others may not have CI artifacts)
 538         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 539         supported_platforms = [
 540             # tier 1
 541             "aarch64-unknown-linux-gnu",
 542             "i686-pc-windows-gnu",
 543             "i686-pc-windows-msvc",
 544             "i686-unknown-linux-gnu",
 545             "x86_64-unknown-linux-gnu",
 546             "x86_64-apple-darwin",
 547             "x86_64-pc-windows-gnu",
 548             "x86_64-pc-windows-msvc",
 549             # tier 2 with host tools
 550             "aarch64-apple-darwin",
 551             "aarch64-pc-windows-msvc",
 552             "aarch64-unknown-linux-musl",
 553             "arm-unknown-linux-gnueabi",
 554             "arm-unknown-linux-gnueabihf",
 555             "armv7-unknown-linux-gnueabihf",
 556             "mips-unknown-linux-gnu",
 557             "mips64-unknown-linux-gnuabi64",
 558             "mips64el-unknown-linux-gnuabi64",
 559             "mipsel-unknown-linux-gnu",
 560             "powerpc-unknown-linux-gnu",
 561             "powerpc64-unknown-linux-gnu",
 562             "powerpc64le-unknown-linux-gnu",
 563             "riscv64gc-unknown-linux-gnu",
 564             "s390x-unknown-linux-gnu",
 565             "x86_64-unknown-freebsd",
 566             "x86_64-unknown-illumos",
 567             "x86_64-unknown-linux-musl",
 568             "x86_64-unknown-netbsd",
 569         ]
 570         return opt == "true" \
 571             or (opt == "if-available" and self.build in supported_platforms)
 572
 573     def _download_component_helper(
 574         self, filename, pattern, tarball_suffix, stage0=True, key=None
 575     ):
 576         if key is None:
 577             if stage0:
 578                 key = self.stage0_compiler.date
 579             else:
 580                 key = self.rustc_commit
 581         cache_dst = os.path.join(self.build_dir, "cache")
 582         rustc_cache = os.path.join(cache_dst, key)
 583         if not os.path.exists(rustc_cache):
 584             os.makedirs(rustc_cache)
 585
 586         if stage0:
 587             base = self._download_url
 588             url = "dist/{}".format(key)
 589         else:
 590             base = "https://ci-artifacts.rust-lang.org"
 591             url = "rustc-builds/{}".format(self.rustc_commit)
 592         tarball = os.path.join(rustc_cache, filename)
 593         if not os.path.exists(tarball):
 594             get(
 595                 base,
 596                 "{}/{}".format(url, filename),
 597                 tarball,
 598                 self.checksums_sha256,
 599                 verbose=self.verbose,
 600                 do_verify=stage0,
 601             )
 602         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 603
 604     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 605         if not llvm_sha:
 606             print("error: could not find commit hash for downloading LLVM")
 607             print("help: maybe your repository history is too shallow?")
 608             print("help: consider disabling `download-ci-llvm`")
 609             print("help: or fetch enough history to include one upstream commit")
 610             exit(1)
 611         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 612         cache_dst = os.path.join(self.build_dir, "cache")
 613         rustc_cache = os.path.join(cache_dst, cache_prefix)
 614         if not os.path.exists(rustc_cache):
 615             os.makedirs(rustc_cache)
 616
 617         base = "https://ci-artifacts.rust-lang.org"
 618         url = "rustc-builds/{}".format(llvm_sha)
 619         if llvm_assertions:
 620             url = url.replace('rustc-builds', 'rustc-builds-alt')
 621         # ci-artifacts are only stored as .xz, not .gz
 622         if not support_xz():
 623             print("error: XZ support is required to download LLVM")
 624             print("help: consider disabling `download-ci-llvm` or using python3")
 625             exit(1)
 626         tarball_suffix = '.tar.xz'
 627         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 628         tarball = os.path.join(rustc_cache, filename)
 629         if not os.path.exists(tarball):
 630             help_on_error = "error: failed to download llvm from ci"
 631             help_on_error += "\nhelp: old builds get deleted after a certain time"
 632             help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
 633             help_on_error += " disable `download-ci-llvm` in config.toml:"
 634             help_on_error += "\n"
 635             help_on_error += "\n[llvm]"
 636             help_on_error += "\ndownload-ci-llvm = false"
 637             help_on_error += "\n"
 638             get(
 639                 base,
 640                 "{}/{}".format(url, filename),
 641                 tarball,
 642                 self.checksums_sha256,
 643                 verbose=self.verbose,
 644                 do_verify=False,
 645                 help_on_error=help_on_error,
 646             )
 647         unpack(tarball, tarball_suffix, self.llvm_root(),
 648                 match="rust-dev",
 649                 verbose=self.verbose)
 650
 651     def fix_bin_or_dylib(self, fname):
 652         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 653         or the RPATH section, to fix the dynamic library search path
 654
 655         This method is only required on NixOS and uses the PatchELF utility to
 656         change the interpreter/RPATH of ELF executables.
 657
 658         Please see https://nixos.org/patchelf.html for more information
 659         """
 660         default_encoding = sys.getdefaultencoding()
 661         try:
 662             ostype = subprocess.check_output(
 663                 ['uname', '-s']).strip().decode(default_encoding)
 664         except subprocess.CalledProcessError:
 665             return
 666         except OSError as reason:
 667             if getattr(reason, 'winerror', None) is not None:
 668                 return
 669             raise reason
 670
 671         if ostype != "Linux":
 672             return
 673
 674         # If the user has asked binaries to be patched for Nix, then
 675         # don't check for NixOS or `/lib`, just continue to the patching.
 676         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 677             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 678             # The latter one does not exist on NixOS when using tmpfs as root.
 679             try:
 680                 with open("/etc/os-release", "r") as f:
 681                     if not any(line.strip() == "ID=nixos" for line in f):
 682                         return
 683             except FileNotFoundError:
 684                 return
 685             if os.path.exists("/lib"):
 686                 return
 687
 688         # At this point we're pretty sure the user is running NixOS or
 689         # using Nix
 690         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 691         print(nix_os_msg, fname)
 692
 693         # Only build `.nix-deps` once.
 694         nix_deps_dir = self.nix_deps_dir
 695         if not nix_deps_dir:
 696             # Run `nix-build` to "build" each dependency (which will likely reuse
 697             # the existing `/nix/store` copy, or at most download a pre-built copy).
 698             #
 699             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 700             # directory, but still reference the actual `/nix/store` path in the rpath
 701             # as it makes it significantly more robust against changes to the location of
 702             # the `.nix-deps` location.
 703             #
 704             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 705             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 706             # patchelf: Needed for patching ELF binaries (see doc comment above).
 707             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 708             nix_expr = '''
 709             with (import <nixpkgs> {});
 710             symlinkJoin {
 711               name = "rust-stage0-dependencies";
 712               paths = [
 713                 zlib
 714                 patchelf
 715                 stdenv.cc.bintools
 716               ];
 717             }
 718             '''
 719             try:
 720                 subprocess.check_output([
 721                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 722                 ])
 723             except subprocess.CalledProcessError as reason:
 724                 print("warning: failed to call nix-build:", reason)
 725                 return
 726             self.nix_deps_dir = nix_deps_dir
 727
 728         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 729         rpath_entries = [
 730             # Relative default, all binary and dynamic libraries we ship
 731             # appear to have this (even when `../lib` is redundant).
 732             "$ORIGIN/../lib",
 733             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 734         ]
 735         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 736         if not fname.endswith(".so"):
 737             # Finally, set the corret .interp for binaries
 738             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 739                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 740
 741         try:
 742             subprocess.check_output([patchelf] + patchelf_args + [fname])
 743         except subprocess.CalledProcessError as reason:
 744             print("warning: failed to call patchelf:", reason)
 745             return
 746
 747     # If `download-rustc` is set, download the most recent commit with CI artifacts
 748     def maybe_download_ci_toolchain(self):
 749         # If `download-rustc` is not set, default to rebuilding.
 750         download_rustc = self.get_toml("download-rustc", section="rust")
 751         if download_rustc is None or download_rustc == "false":
 752             return None
 753         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 754
 755         # Handle running from a directory other than the top level
 756         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 757         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 758         compiler = "{}/compiler/".format(top_level)
 759         library = "{}/library/".format(top_level)
 760
 761         # Look for a version to compare to based on the current commit.
 762         # Only commits merged by bors will have CI artifacts.
 763         merge_base = [
 764             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 765             "--first-parent", "HEAD"
 766         ]
 767         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 768         if not commit:
 769             print("error: could not find commit hash for downloading rustc")
 770             print("help: maybe your repository history is too shallow?")
 771             print("help: consider disabling `download-rustc`")
 772             print("help: or fetch enough history to include one upstream commit")
 773             exit(1)
 774
 775         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 776         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 777         if status != 0:
 778             if download_rustc == "if-unchanged":
 779                 if self.verbose:
 780                     print("warning: saw changes to compiler/ or library/ since {}; " \
 781                           "ignoring `download-rustc`".format(commit))
 782                 return None
 783             print("warning: `download-rustc` is enabled, but there are changes to " \
 784                   "compiler/ or library/")
 785
 786         if self.verbose:
 787             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
 788         self.rustc_commit = commit
 789         # FIXME: support downloading artifacts from the beta channel
 790         self.download_toolchain(False, "nightly")
 791
 792     def rustc_stamp(self, stage0):
 793         """Return the path for .rustc-stamp at the given stage
 794
 795         >>> rb = RustBuild()
 796         >>> rb.build_dir = "build"
 797         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 798         True
 799         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 800         True
 801         """
 802         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 803
 804     def rustfmt_stamp(self):
 805         """Return the path for .rustfmt-stamp
 806
 807         >>> rb = RustBuild()
 808         >>> rb.build_dir = "build"
 809         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 810         True
 811         """
 812         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 813
 814     def llvm_stamp(self):
 815         """Return the path for .rustfmt-stamp
 816
 817         >>> rb = RustBuild()
 818         >>> rb.build_dir = "build"
 819         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 820         True
 821         """
 822         return os.path.join(self.llvm_root(), '.llvm-stamp')
 823
 824
 825     def program_out_of_date(self, stamp_path, key):
 826         """Check if the given program stamp is out of date"""
 827         if not os.path.exists(stamp_path) or self.clean:
 828             return True
 829         with open(stamp_path, 'r') as stamp:
 830             return key != stamp.read()
 831
 832     def bin_root(self, stage0):
 833         """Return the binary root directory for the given stage
 834
 835         >>> rb = RustBuild()
 836         >>> rb.build_dir = "build"
 837         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 838         True
 839         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 840         True
 841
 842         When the 'build' property is given should be a nested directory:
 843
 844         >>> rb.build = "devel"
 845         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 846         True
 847         """
 848         if stage0:
 849             subdir = "stage0"
 850         else:
 851             subdir = "ci-rustc"
 852         return os.path.join(self.build_dir, self.build, subdir)
 853
 854     def llvm_root(self):
 855         """Return the CI LLVM root directory
 856
 857         >>> rb = RustBuild()
 858         >>> rb.build_dir = "build"
 859         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 860         True
 861
 862         When the 'build' property is given should be a nested directory:
 863
 864         >>> rb.build = "devel"
 865         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 866         True
 867         """
 868         return os.path.join(self.build_dir, self.build, "ci-llvm")
 869
 870     def get_toml(self, key, section=None):
 871         """Returns the value of the given key in config.toml, otherwise returns None
 872
 873         >>> rb = RustBuild()
 874         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 875         >>> rb.get_toml("key2")
 876         'value2'
 877
 878         If the key does not exist, the result is None:
 879
 880         >>> rb.get_toml("key3") is None
 881         True
 882
 883         Optionally also matches the section the key appears in
 884
 885         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 886         >>> rb.get_toml('key', 'a')
 887         'value1'
 888         >>> rb.get_toml('key', 'b')
 889         'value2'
 890         >>> rb.get_toml('key', 'c') is None
 891         True
 892
 893         >>> rb.config_toml = 'key1 = true'
 894         >>> rb.get_toml("key1")
 895         'true'
 896         """
 897
 898         cur_section = None
 899         for line in self.config_toml.splitlines():
 900             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 901             if section_match is not None:
 902                 cur_section = section_match.group(1)
 903
 904             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 905             if match is not None:
 906                 value = match.group(1)
 907                 if section is None or section == cur_section:
 908                     return self.get_string(value) or value.strip()
 909         return None
 910
 911     def cargo(self):
 912         """Return config path for cargo"""
 913         return self.program_config('cargo')
 914
 915     def rustc(self, stage0):
 916         """Return config path for rustc"""
 917         return self.program_config('rustc', stage0)
 918
 919     def rustfmt(self):
 920         """Return config path for rustfmt"""
 921         if self.stage0_rustfmt is None:
 922             return None
 923         return self.program_config('rustfmt')
 924
 925     def program_config(self, program, stage0=True):
 926         """Return config path for the given program at the given stage
 927
 928         >>> rb = RustBuild()
 929         >>> rb.config_toml = 'rustc = "rustc"\\n'
 930         >>> rb.program_config('rustc')
 931         'rustc'
 932         >>> rb.config_toml = ''
 933         >>> cargo_path = rb.program_config('cargo', True)
 934         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 935         ... "bin", "cargo")
 936         True
 937         >>> cargo_path = rb.program_config('cargo', False)
 938         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 939         ... "bin", "cargo")
 940         True
 941         """
 942         config = self.get_toml(program)
 943         if config:
 944             return os.path.expanduser(config)
 945         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 946             program, self.exe_suffix()))
 947
 948     @staticmethod
 949     def get_string(line):
 950         """Return the value between double quotes
 951
 952         >>> RustBuild.get_string('    "devel"   ')
 953         'devel'
 954         >>> RustBuild.get_string("    'devel'   ")
 955         'devel'
 956         >>> RustBuild.get_string('devel') is None
 957         True
 958         >>> RustBuild.get_string('    "devel   ')
 959         ''
 960         """
 961         start = line.find('"')
 962         if start != -1:
 963             end = start + 1 + line[start + 1:].find('"')
 964             return line[start + 1:end]
 965         start = line.find('\'')
 966         if start != -1:
 967             end = start + 1 + line[start + 1:].find('\'')
 968             return line[start + 1:end]
 969         return None
 970
 971     @staticmethod
 972     def exe_suffix():
 973         """Return a suffix for executables"""
 974         if sys.platform == 'win32':
 975             return '.exe'
 976         return ''
 977
 978     def bootstrap_binary(self):
 979         """Return the path of the bootstrap binary
 980
 981         >>> rb = RustBuild()
 982         >>> rb.build_dir = "build"
 983         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 984         ... "debug", "bootstrap")
 985         True
 986         """
 987         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 988
 989     def build_bootstrap(self):
 990         """Build bootstrap"""
 991         print("Building rustbuild")
 992         build_dir = os.path.join(self.build_dir, "bootstrap")
 993         if self.clean and os.path.exists(build_dir):
 994             shutil.rmtree(build_dir)
 995         env = os.environ.copy()
 996         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 997         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 998         if "CARGO_BUILD_TARGET" in env:
 999             del env["CARGO_BUILD_TARGET"]
1000         env["CARGO_TARGET_DIR"] = build_dir
1001         env["RUSTC"] = self.rustc(True)
1002         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1003             (os.pathsep + env["LD_LIBRARY_PATH"]) \
1004             if "LD_LIBRARY_PATH" in env else ""
1005         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1006             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
1007             if "DYLD_LIBRARY_PATH" in env else ""
1008         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1009             (os.pathsep + env["LIBRARY_PATH"]) \
1010             if "LIBRARY_PATH" in env else ""
1011
1012         # preserve existing RUSTFLAGS
1013         env.setdefault("RUSTFLAGS", "")
1014         build_section = "target.{}".format(self.build)
1015         target_features = []
1016         if self.get_toml("crt-static", build_section) == "true":
1017             target_features += ["+crt-static"]
1018         elif self.get_toml("crt-static", build_section) == "false":
1019             target_features += ["-crt-static"]
1020         if target_features:
1021             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1022         target_linker = self.get_toml("linker", build_section)
1023         if target_linker is not None:
1024             env["RUSTFLAGS"] += " -C linker=" + target_linker
1025         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1026         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1027         if self.get_toml("deny-warnings", "rust") != "false":
1028             env["RUSTFLAGS"] += " -Dwarnings"
1029
1030         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1031             os.pathsep + env["PATH"]
1032         if not os.path.isfile(self.cargo()):
1033             raise Exception("no cargo executable found at `{}`".format(
1034                 self.cargo()))
1035         args = [self.cargo(), "build", "--manifest-path",
1036                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1037         for _ in range(0, self.verbose):
1038             args.append("--verbose")
1039         if self.use_locked_deps:
1040             args.append("--locked")
1041         if self.use_vendored_sources:
1042             args.append("--frozen")
1043         run(args, env=env, verbose=self.verbose)
1044
1045     def build_triple(self):
1046         """Build triple as in LLVM
1047
1048         Note that `default_build_triple` is moderately expensive,
1049         so use `self.build` where possible.
1050         """
1051         config = self.get_toml('build')
1052         if config:
1053             return config
1054         return default_build_triple(self.verbose)
1055
1056     def check_submodule(self, module, slow_submodules):
1057         if not slow_submodules:
1058             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1059                                            cwd=os.path.join(self.rust_root, module),
1060                                            stdout=subprocess.PIPE)
1061             return checked_out
1062         else:
1063             return None
1064
1065     def update_submodule(self, module, checked_out, recorded_submodules):
1066         module_path = os.path.join(self.rust_root, module)
1067
1068         if checked_out is not None:
1069             default_encoding = sys.getdefaultencoding()
1070             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1071             if recorded_submodules[module] == checked_out:
1072                 return
1073
1074         print("Updating submodule", module)
1075
1076         run(["git", "submodule", "-q", "sync", module],
1077             cwd=self.rust_root, verbose=self.verbose)
1078
1079         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1080         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1081             update_args.append("--progress")
1082         update_args.append(module)
1083         try:
1084             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1085         except RuntimeError:
1086             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1087             print('Either stash the changes by running "git stash" within the submodule\'s')
1088             print('directory, reset them by running "git reset --hard", or commit them.')
1089             print("To reset all submodules' changes run", end=" ")
1090             print('"git submodule foreach --recursive git reset --hard".')
1091             raise SystemExit(1)
1092
1093         run(["git", "reset", "-q", "--hard"],
1094             cwd=module_path, verbose=self.verbose)
1095         run(["git", "clean", "-qdfx"],
1096             cwd=module_path, verbose=self.verbose)
1097
1098     def update_submodules(self):
1099         """Update submodules"""
1100         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1101                 self.get_toml('submodules') == "false":
1102             return
1103
1104         default_encoding = sys.getdefaultencoding()
1105
1106         # check the existence and version of 'git' command
1107         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1108         self.git_version = distutils.version.LooseVersion(git_version_str)
1109
1110         slow_submodules = self.get_toml('fast-submodules') == "false"
1111         start_time = time()
1112         if slow_submodules:
1113             print('Unconditionally updating submodules')
1114         else:
1115             print('Updating only changed submodules')
1116         default_encoding = sys.getdefaultencoding()
1117         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1118         # currently requires everything in a workspace to be "locally present" when starting a
1119         # build, and will give a hard error if any Cargo.toml files are missing.
1120         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1121         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1122         #   That will still require cloning the submodules the second you check the standard
1123         #   library, though...
1124         # FIXME: Is there a way to avoid hard-coding the submodules required?
1125         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1126         submodules = [
1127             "src/tools/rust-installer",
1128             "src/tools/cargo",
1129             "src/tools/rls",
1130             "src/tools/miri",
1131             "library/backtrace",
1132             "library/stdarch"
1133         ]
1134         filtered_submodules = []
1135         submodules_names = []
1136         for module in submodules:
1137             check = self.check_submodule(module, slow_submodules)
1138             filtered_submodules.append((module, check))
1139             submodules_names.append(module)
1140         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1141                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1142         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1143         # { filename: hash }
1144         recorded_submodules = {}
1145         for data in recorded:
1146             # [mode, kind, hash, filename]
1147             data = data.split()
1148             recorded_submodules[data[3]] = data[2]
1149         for module in filtered_submodules:
1150             self.update_submodule(module[0], module[1], recorded_submodules)
1151         print("  Submodules updated in %.2f seconds" % (time() - start_time))
1152
1153     def set_dist_environment(self, url):
1154         """Set download URL for normal environment"""
1155         if 'RUSTUP_DIST_SERVER' in os.environ:
1156             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1157         else:
1158             self._download_url = url
1159
1160     def check_vendored_status(self):
1161         """Check that vendoring is configured properly"""
1162         vendor_dir = os.path.join(self.rust_root, 'vendor')
1163         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1164             if os.environ.get('USER') != os.environ['SUDO_USER']:
1165                 self.use_vendored_sources = True
1166                 print('info: looks like you are running this command under `sudo`')
1167                 print('      and so in order to preserve your $HOME this will now')
1168                 print('      use vendored sources by default.')
1169                 if not os.path.exists(vendor_dir):
1170                     print('error: vendoring required, but vendor directory does not exist.')
1171                     print('       Run `cargo vendor` without sudo to initialize the '
1172                           'vendor directory.')
1173                     raise Exception("{} not found".format(vendor_dir))
1174
1175         if self.use_vendored_sources:
1176             config = ("[source.crates-io]\n"
1177                       "replace-with = 'vendored-sources'\n"
1178                       "registry = 'https://example.com'\n"
1179                       "\n"
1180                       "[source.vendored-sources]\n"
1181                       "directory = '{}/vendor'\n"
1182                       .format(self.rust_root))
1183             if not os.path.exists('.cargo'):
1184                 os.makedirs('.cargo')
1185                 with output('.cargo/config') as cargo_config:
1186                     cargo_config.write(config)
1187             else:
1188                 print('info: using vendored source, but .cargo/config is already present.')
1189                 print('      Reusing the current configuration file. But you may want to '
1190                       'configure vendoring like this:')
1191                 print(config)
1192         else:
1193             if os.path.exists('.cargo'):
1194                 shutil.rmtree('.cargo')
1195
1196     def ensure_vendored(self):
1197         """Ensure that the vendored sources are available if needed"""
1198         vendor_dir = os.path.join(self.rust_root, 'vendor')
1199         # Note that this does not handle updating the vendored dependencies if
1200         # the rust git repository is updated. Normal development usually does
1201         # not use vendoring, so hopefully this isn't too much of a problem.
1202         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1203             run([
1204                 self.cargo(),
1205                 "vendor",
1206                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1207                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1208             ], verbose=self.verbose, cwd=self.rust_root)
1209
1210
1211 def bootstrap(help_triggered):
1212     """Configure, fetch, build and run the initial bootstrap"""
1213
1214     # If the user is asking for help, let them know that the whole download-and-build
1215     # process has to happen before anything is printed out.
1216     if help_triggered:
1217         print("info: Downloading and building bootstrap before processing --help")
1218         print("      command. See src/bootstrap/README.md for help with common")
1219         print("      commands.")
1220
1221     parser = argparse.ArgumentParser(description='Build rust')
1222     parser.add_argument('--config')
1223     parser.add_argument('--build')
1224     parser.add_argument('--clean', action='store_true')
1225     parser.add_argument('-v', '--verbose', action='count', default=0)
1226
1227     args = [a for a in sys.argv if a != '-h' and a != '--help']
1228     args, _ = parser.parse_known_args(args)
1229
1230     # Configure initial bootstrap
1231     build = RustBuild()
1232     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1233     build.verbose = args.verbose
1234     build.clean = args.clean
1235
1236     # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1237     # then `config.toml` in the root directory.
1238     toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1239     using_default_path = toml_path is None
1240     if using_default_path:
1241         toml_path = 'config.toml'
1242         if not os.path.exists(toml_path):
1243             toml_path = os.path.join(build.rust_root, toml_path)
1244
1245     # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1246     # but not if `config.toml` hasn't been created.
1247     if not using_default_path or os.path.exists(toml_path):
1248         with open(toml_path) as config:
1249             build.config_toml = config.read()
1250
1251     profile = build.get_toml('profile')
1252     if profile is not None:
1253         include_file = 'config.{}.toml'.format(profile)
1254         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1255         include_path = os.path.join(include_dir, include_file)
1256         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1257         # specific key, so appending our defaults at the end allows the user to override them
1258         with open(include_path) as included_toml:
1259             build.config_toml += os.linesep + included_toml.read()
1260
1261     config_verbose = build.get_toml('verbose', 'build')
1262     if config_verbose is not None:
1263         build.verbose = max(build.verbose, int(config_verbose))
1264
1265     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1266
1267     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1268
1269     build.check_vendored_status()
1270
1271     build_dir = build.get_toml('build-dir', 'build') or 'build'
1272     build.build_dir = os.path.abspath(build_dir)
1273
1274     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1275         data = json.load(f)
1276     build.checksums_sha256 = data["checksums_sha256"]
1277     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1278     if data.get("rustfmt") is not None:
1279         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1280
1281     build.set_dist_environment(data["dist_server"])
1282
1283     build.build = args.build or build.build_triple()
1284
1285     # Acquire the lock before doing any build actions
1286     # The lock is released when `lock` is dropped
1287     if not os.path.exists(build.build_dir):
1288         os.makedirs(build.build_dir)
1289     lock = acquire_lock(build.build_dir)
1290     build.update_submodules()
1291
1292     # Fetch/build the bootstrap
1293     build.download_toolchain()
1294     # Download the master compiler if `download-rustc` is set
1295     build.maybe_download_ci_toolchain()
1296     sys.stdout.flush()
1297     build.ensure_vendored()
1298     build.build_bootstrap()
1299     sys.stdout.flush()
1300
1301     # Run the bootstrap
1302     args = [build.bootstrap_binary()]
1303     args.extend(sys.argv[1:])
1304     env = os.environ.copy()
1305     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1306     env["BOOTSTRAP_PYTHON"] = sys.executable
1307     env["RUSTC_BOOTSTRAP"] = '1'
1308     if build.rustc_commit is not None:
1309         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1310     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1311
1312
1313 def main():
1314     """Entry point for the bootstrap process"""
1315     start_time = time()
1316
1317     # x.py help <cmd> ...
1318     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1319         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1320
1321     help_triggered = (
1322         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1323     try:
1324         bootstrap(help_triggered)
1325         if not help_triggered:
1326             print("Build completed successfully in {}".format(
1327                 format_build_time(time() - start_time)))
1328     except (SystemExit, KeyboardInterrupt) as error:
1329         if hasattr(error, 'code') and isinstance(error.code, int):
1330             exit_code = error.code
1331         else:
1332             exit_code = 1
1333             print(error)
1334         if not help_triggered:
1335             print("Build completed unsuccessfully in {}".format(
1336                 format_build_time(time() - start_time)))
1337         sys.exit(exit_code)
1338
1339
1340 if __name__ == '__main__':
1341     main()