src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import json
   8 import os
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tarfile
  14 import tempfile
  15
  16 from time import time, sleep
  17
  18 # Acquire a lock on the build directory to make sure that
  19 # we don't cause a race condition while building
  20 # Lock is created in `build_dir/lock.db`
  21 def acquire_lock(build_dir):
  22     try:
  23         import sqlite3
  24
  25         path = os.path.join(build_dir, "lock.db")
  26         try:
  27             con = sqlite3.Connection(path, timeout=0)
  28             curs = con.cursor()
  29             curs.execute("BEGIN EXCLUSIVE")
  30             # The lock is released when the cursor is dropped
  31             return curs
  32         # If the database is busy then lock has already been acquired
  33         # so we wait for the lock.
  34         # We retry every quarter second so that execution is passed back to python
  35         # so that it can handle signals
  36         except sqlite3.OperationalError:
  37             del con
  38             del curs
  39             print("Waiting for lock on build directory")
  40             con = sqlite3.Connection(path, timeout=0.25)
  41             curs = con.cursor()
  42             while True:
  43                 try:
  44                     curs.execute("BEGIN EXCLUSIVE")
  45                     break
  46                 except sqlite3.OperationalError:
  47                     pass
  48                 sleep(0.25)
  49             return curs
  50     except ImportError:
  51         print("warning: sqlite3 not available in python, skipping build directory lock")
  52         print("please file an issue on rust-lang/rust")
  53         print("this is not a problem for non-concurrent x.py invocations")
  54         return None
  55
  56 def support_xz():
  57     try:
  58         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  59             temp_path = temp_file.name
  60         with tarfile.open(temp_path, "w:xz"):
  61             pass
  62         return True
  63     except tarfile.CompressionError:
  64         return False
  65
  66 def get(base, url, path, checksums, verbose=False, do_verify=True):
  67     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  68         temp_path = temp_file.name
  69
  70     try:
  71         if do_verify:
  72             if url not in checksums:
  73                 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
  74             sha256 = checksums[url]
  75             if os.path.exists(path):
  76                 if verify(path, sha256, False):
  77                     if verbose:
  78                         print("using already-download file", path)
  79                     return
  80                 else:
  81                     if verbose:
  82                         print("ignoring already-download file",
  83                             path, "due to failed verification")
  84                     os.unlink(path)
  85         download(temp_path, "{}/{}".format(base, url), True, verbose)
  86         if do_verify and not verify(temp_path, sha256, verbose):
  87             raise RuntimeError("failed verification")
  88         if verbose:
  89             print("moving {} to {}".format(temp_path, path))
  90         shutil.move(temp_path, path)
  91     finally:
  92         if os.path.isfile(temp_path):
  93             if verbose:
  94                 print("removing", temp_path)
  95             os.unlink(temp_path)
  96
  97
  98 def download(path, url, probably_big, verbose):
  99     for _ in range(0, 4):
 100         try:
 101             _download(path, url, probably_big, verbose, True)
 102             return
 103         except RuntimeError:
 104             print("\nspurious failure, trying again")
 105     _download(path, url, probably_big, verbose, False)
 106
 107
 108 def _download(path, url, probably_big, verbose, exception):
 109     if probably_big or verbose:
 110         print("downloading {}".format(url))
 111     # see https://serverfault.com/questions/301128/how-to-download
 112     if sys.platform == 'win32':
 113         run(["PowerShell.exe", "/nologo", "-Command",
 114              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
 115              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
 116             verbose=verbose,
 117             exception=exception)
 118     else:
 119         if probably_big or verbose:
 120             option = "-#"
 121         else:
 122             option = "-s"
 123         require(["curl", "--version"])
 124         run(["curl", option,
 125              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
 126              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
 127              "--retry", "3", "-Sf", "-o", path, url],
 128             verbose=verbose,
 129             exception=exception)
 130
 131
 132 def verify(path, expected, verbose):
 133     """Check if the sha256 sum of the given path is valid"""
 134     if verbose:
 135         print("verifying", path)
 136     with open(path, "rb") as source:
 137         found = hashlib.sha256(source.read()).hexdigest()
 138     verified = found == expected
 139     if not verified:
 140         print("invalid checksum:\n"
 141               "    found:    {}\n"
 142               "    expected: {}".format(found, expected))
 143     return verified
 144
 145
 146 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 147     """Unpack the given tarball file"""
 148     print("extracting", tarball)
 149     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 150     with contextlib.closing(tarfile.open(tarball)) as tar:
 151         for member in tar.getnames():
 152             if "/" not in member:
 153                 continue
 154             name = member.replace(fname + "/", "", 1)
 155             if match is not None and not name.startswith(match):
 156                 continue
 157             name = name[len(match) + 1:]
 158
 159             dst_path = os.path.join(dst, name)
 160             if verbose:
 161                 print("  extracting", member)
 162             tar.extract(member, dst)
 163             src_path = os.path.join(dst, member)
 164             if os.path.isdir(src_path) and os.path.exists(dst_path):
 165                 continue
 166             shutil.move(src_path, dst_path)
 167     shutil.rmtree(os.path.join(dst, fname))
 168
 169
 170 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
 171     """Run a child program in a new process"""
 172     if verbose:
 173         print("running: " + ' '.join(args))
 174     sys.stdout.flush()
 175     # Use Popen here instead of call() as it apparently allows powershell on
 176     # Windows to not lock up waiting for input presumably.
 177     ret = subprocess.Popen(args, **kwargs)
 178     code = ret.wait()
 179     if code != 0:
 180         err = "failed to run: " + ' '.join(args)
 181         if verbose or exception:
 182             raise RuntimeError(err)
 183         # For most failures, we definitely do want to print this error, or the user will have no
 184         # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
 185         # have already printed an error above, so there's no need to print the exact command we're
 186         # running.
 187         if is_bootstrap:
 188             sys.exit(1)
 189         else:
 190             sys.exit(err)
 191
 192
 193 def require(cmd, exit=True):
 194     '''Run a command, returning its output.
 195     On error,
 196         If `exit` is `True`, exit the process.
 197         Otherwise, return None.'''
 198     try:
 199         return subprocess.check_output(cmd).strip()
 200     except (subprocess.CalledProcessError, OSError) as exc:
 201         if not exit:
 202             return None
 203         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 204         print("Please make sure it's installed and in the path.")
 205         sys.exit(1)
 206
 207
 208 def format_build_time(duration):
 209     """Return a nicer format for build time
 210
 211     >>> format_build_time('300')
 212     '0:05:00'
 213     """
 214     return str(datetime.timedelta(seconds=int(duration)))
 215
 216
 217 def default_build_triple(verbose):
 218     """Build triple as in LLVM"""
 219     # If the user already has a host build triple with an existing `rustc`
 220     # install, use their preference. This fixes most issues with Windows builds
 221     # being detected as GNU instead of MSVC.
 222     default_encoding = sys.getdefaultencoding()
 223     try:
 224         version = subprocess.check_output(["rustc", "--version", "--verbose"],
 225                 stderr=subprocess.DEVNULL)
 226         version = version.decode(default_encoding)
 227         host = next(x for x in version.split('\n') if x.startswith("host: "))
 228         triple = host.split("host: ")[1]
 229         if verbose:
 230             print("detected default triple {} from pre-installed rustc".format(triple))
 231         return triple
 232     except Exception as e:
 233         if verbose:
 234             print("pre-installed rustc not detected: {}".format(e))
 235             print("falling back to auto-detect")
 236
 237     required = sys.platform != 'win32'
 238     ostype = require(["uname", "-s"], exit=required)
 239     cputype = require(['uname', '-m'], exit=required)
 240
 241     # If we do not have `uname`, assume Windows.
 242     if ostype is None or cputype is None:
 243         return 'x86_64-pc-windows-msvc'
 244
 245     ostype = ostype.decode(default_encoding)
 246     cputype = cputype.decode(default_encoding)
 247
 248     # The goal here is to come up with the same triple as LLVM would,
 249     # at least for the subset of platforms we're willing to target.
 250     ostype_mapper = {
 251         'Darwin': 'apple-darwin',
 252         'DragonFly': 'unknown-dragonfly',
 253         'FreeBSD': 'unknown-freebsd',
 254         'Haiku': 'unknown-haiku',
 255         'NetBSD': 'unknown-netbsd',
 256         'OpenBSD': 'unknown-openbsd'
 257     }
 258
 259     # Consider the direct transformation first and then the special cases
 260     if ostype in ostype_mapper:
 261         ostype = ostype_mapper[ostype]
 262     elif ostype == 'Linux':
 263         os_from_sp = subprocess.check_output(
 264             ['uname', '-o']).strip().decode(default_encoding)
 265         if os_from_sp == 'Android':
 266             ostype = 'linux-android'
 267         else:
 268             ostype = 'unknown-linux-gnu'
 269     elif ostype == 'SunOS':
 270         ostype = 'pc-solaris'
 271         # On Solaris, uname -m will return a machine classification instead
 272         # of a cpu type, so uname -p is recommended instead.  However, the
 273         # output from that option is too generic for our purposes (it will
 274         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 275         # must be used instead.
 276         cputype = require(['isainfo', '-k']).decode(default_encoding)
 277         # sparc cpus have sun as a target vendor
 278         if 'sparc' in cputype:
 279             ostype = 'sun-solaris'
 280     elif ostype.startswith('MINGW'):
 281         # msys' `uname` does not print gcc configuration, but prints msys
 282         # configuration. so we cannot believe `uname -m`:
 283         # msys1 is always i686 and msys2 is always x86_64.
 284         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 285         # MINGW64 on x86_64.
 286         ostype = 'pc-windows-gnu'
 287         cputype = 'i686'
 288         if os.environ.get('MSYSTEM') == 'MINGW64':
 289             cputype = 'x86_64'
 290     elif ostype.startswith('MSYS'):
 291         ostype = 'pc-windows-gnu'
 292     elif ostype.startswith('CYGWIN_NT'):
 293         cputype = 'i686'
 294         if ostype.endswith('WOW64'):
 295             cputype = 'x86_64'
 296         ostype = 'pc-windows-gnu'
 297     elif sys.platform == 'win32':
 298         # Some Windows platforms might have a `uname` command that returns a
 299         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 300         # these cases, fall back to using sys.platform.
 301         return 'x86_64-pc-windows-msvc'
 302     else:
 303         err = "unknown OS type: {}".format(ostype)
 304         sys.exit(err)
 305
 306     if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
 307         cputype = subprocess.check_output(
 308               ['uname', '-p']).strip().decode(default_encoding)
 309     cputype_mapper = {
 310         'BePC': 'i686',
 311         'aarch64': 'aarch64',
 312         'amd64': 'x86_64',
 313         'arm64': 'aarch64',
 314         'i386': 'i686',
 315         'i486': 'i686',
 316         'i686': 'i686',
 317         'i786': 'i686',
 318         'm68k': 'm68k',
 319         'powerpc': 'powerpc',
 320         'powerpc64': 'powerpc64',
 321         'powerpc64le': 'powerpc64le',
 322         'ppc': 'powerpc',
 323         'ppc64': 'powerpc64',
 324         'ppc64le': 'powerpc64le',
 325         'riscv64': 'riscv64gc',
 326         's390x': 's390x',
 327         'x64': 'x86_64',
 328         'x86': 'i686',
 329         'x86-64': 'x86_64',
 330         'x86_64': 'x86_64'
 331     }
 332
 333     # Consider the direct transformation first and then the special cases
 334     if cputype in cputype_mapper:
 335         cputype = cputype_mapper[cputype]
 336     elif cputype in {'xscale', 'arm'}:
 337         cputype = 'arm'
 338         if ostype == 'linux-android':
 339             ostype = 'linux-androideabi'
 340         elif ostype == 'unknown-freebsd':
 341             cputype = subprocess.check_output(
 342                 ['uname', '-p']).strip().decode(default_encoding)
 343             ostype = 'unknown-freebsd'
 344     elif cputype == 'armv6l':
 345         cputype = 'arm'
 346         if ostype == 'linux-android':
 347             ostype = 'linux-androideabi'
 348         else:
 349             ostype += 'eabihf'
 350     elif cputype in {'armv7l', 'armv8l'}:
 351         cputype = 'armv7'
 352         if ostype == 'linux-android':
 353             ostype = 'linux-androideabi'
 354         else:
 355             ostype += 'eabihf'
 356     elif cputype == 'mips':
 357         if sys.byteorder == 'big':
 358             cputype = 'mips'
 359         elif sys.byteorder == 'little':
 360             cputype = 'mipsel'
 361         else:
 362             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 363     elif cputype == 'mips64':
 364         if sys.byteorder == 'big':
 365             cputype = 'mips64'
 366         elif sys.byteorder == 'little':
 367             cputype = 'mips64el'
 368         else:
 369             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 370         # only the n64 ABI is supported, indicate it
 371         ostype += 'abi64'
 372     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 373         pass
 374     else:
 375         err = "unknown cpu type: {}".format(cputype)
 376         sys.exit(err)
 377
 378     return "{}-{}".format(cputype, ostype)
 379
 380
 381 @contextlib.contextmanager
 382 def output(filepath):
 383     tmp = filepath + '.tmp'
 384     with open(tmp, 'w') as f:
 385         yield f
 386     try:
 387         if os.path.exists(filepath):
 388             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 389     except OSError:
 390         shutil.copy2(tmp, filepath)
 391         os.remove(tmp)
 392         return
 393     os.rename(tmp, filepath)
 394
 395
 396 class Stage0Toolchain:
 397     def __init__(self, stage0_payload):
 398         self.date = stage0_payload["date"]
 399         self.version = stage0_payload["version"]
 400
 401     def channel(self):
 402         return self.version + "-" + self.date
 403
 404
 405 class RustBuild(object):
 406     """Provide all the methods required to build Rust"""
 407     def __init__(self):
 408         self.checksums_sha256 = {}
 409         self.stage0_compiler = None
 410         self.stage0_rustfmt = None
 411         self._download_url = ''
 412         self.build = ''
 413         self.build_dir = ''
 414         self.clean = False
 415         self.config_toml = ''
 416         self.rust_root = ''
 417         self.use_locked_deps = ''
 418         self.use_vendored_sources = ''
 419         self.verbose = False
 420         self.git_version = None
 421         self.nix_deps_dir = None
 422         self.rustc_commit = None
 423
 424     def download_toolchain(self, stage0=True, rustc_channel=None):
 425         """Fetch the build system for Rust, written in Rust
 426
 427         This method will build a cache directory, then it will fetch the
 428         tarball which has the stage0 compiler used to then bootstrap the Rust
 429         compiler itself.
 430
 431         Each downloaded tarball is extracted, after that, the script
 432         will move all the content to the right place.
 433         """
 434         if rustc_channel is None:
 435             rustc_channel = self.stage0_compiler.version
 436         bin_root = self.bin_root(stage0)
 437
 438         key = self.stage0_compiler.date
 439         if not stage0:
 440             key += str(self.rustc_commit)
 441         if self.rustc(stage0).startswith(bin_root) and \
 442                 (not os.path.exists(self.rustc(stage0)) or
 443                  self.program_out_of_date(self.rustc_stamp(stage0), key)):
 444             if os.path.exists(bin_root):
 445                 shutil.rmtree(bin_root)
 446             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 447             filename = "rust-std-{}-{}{}".format(
 448                 rustc_channel, self.build, tarball_suffix)
 449             pattern = "rust-std-{}".format(self.build)
 450             self._download_component_helper(filename, pattern, tarball_suffix, stage0)
 451             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 452                                               tarball_suffix)
 453             self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
 454             # download-rustc doesn't need its own cargo, it can just use beta's.
 455             if stage0:
 456                 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 457                                                 tarball_suffix)
 458                 self._download_component_helper(filename, "cargo", tarball_suffix)
 459                 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
 460             else:
 461                 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
 462                 self._download_component_helper(
 463                     filename, "rustc-dev", tarball_suffix, stage0
 464                 )
 465
 466             self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
 467             self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
 468             lib_dir = "{}/lib".format(bin_root)
 469             for lib in os.listdir(lib_dir):
 470                 if lib.endswith(".so"):
 471                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
 472             with output(self.rustc_stamp(stage0)) as rust_stamp:
 473                 rust_stamp.write(key)
 474
 475         if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
 476             not os.path.exists(self.rustfmt())
 477             or self.program_out_of_date(
 478                 self.rustfmt_stamp(),
 479                 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
 480             )
 481         ):
 482             if self.stage0_rustfmt is not None:
 483                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 484                 filename = "rustfmt-{}-{}{}".format(
 485                     self.stage0_rustfmt.version, self.build, tarball_suffix,
 486                 )
 487                 self._download_component_helper(
 488                     filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
 489                 )
 490                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
 491                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
 492                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 493                     rustfmt_stamp.write(self.stage0_rustfmt.channel())
 494
 495         # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
 496         if self.downloading_llvm() and stage0:
 497             # We want the most recent LLVM submodule update to avoid downloading
 498             # LLVM more often than necessary.
 499             #
 500             # This git command finds that commit SHA, looking for bors-authored
 501             # commits that modified src/llvm-project or other relevant version
 502             # stamp files.
 503             #
 504             # This works even in a repository that has not yet initialized
 505             # submodules.
 506             top_level = subprocess.check_output([
 507                 "git", "rev-parse", "--show-toplevel",
 508             ]).decode(sys.getdefaultencoding()).strip()
 509             llvm_sha = subprocess.check_output([
 510                 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 511                 "--first-parent", "HEAD",
 512                 "--",
 513                 "{}/src/llvm-project".format(top_level),
 514                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 515                 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
 516                 "{}/src/version".format(top_level)
 517             ]).decode(sys.getdefaultencoding()).strip()
 518             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 519             llvm_root = self.llvm_root()
 520             llvm_lib = os.path.join(llvm_root, "lib")
 521             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 522                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 523                 for binary in ["llvm-config", "FileCheck"]:
 524                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
 525                 for lib in os.listdir(llvm_lib):
 526                     if lib.endswith(".so"):
 527                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
 528                 with output(self.llvm_stamp()) as llvm_stamp:
 529                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 530
 531     def downloading_llvm(self):
 532         opt = self.get_toml('download-ci-llvm', 'llvm')
 533         # This is currently all tier 1 targets and tier 2 targets with host tools
 534         # (since others may not have CI artifacts)
 535         # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
 536         supported_platforms = [
 537             # tier 1
 538             "aarch64-unknown-linux-gnu",
 539             "i686-pc-windows-gnu",
 540             "i686-pc-windows-msvc",
 541             "i686-unknown-linux-gnu",
 542             "x86_64-unknown-linux-gnu",
 543             "x86_64-apple-darwin",
 544             "x86_64-pc-windows-gnu",
 545             "x86_64-pc-windows-msvc",
 546             # tier 2 with host tools
 547             "aarch64-apple-darwin",
 548             "aarch64-pc-windows-msvc",
 549             "aarch64-unknown-linux-musl",
 550             "arm-unknown-linux-gnueabi",
 551             "arm-unknown-linux-gnueabihf",
 552             "armv7-unknown-linux-gnueabihf",
 553             "mips-unknown-linux-gnu",
 554             "mips64-unknown-linux-gnuabi64",
 555             "mips64el-unknown-linux-gnuabi64",
 556             "mipsel-unknown-linux-gnu",
 557             "powerpc-unknown-linux-gnu",
 558             "powerpc64-unknown-linux-gnu",
 559             "powerpc64le-unknown-linux-gnu",
 560             "riscv64gc-unknown-linux-gnu",
 561             "s390x-unknown-linux-gnu",
 562             "x86_64-unknown-freebsd",
 563             "x86_64-unknown-illumos",
 564             "x86_64-unknown-linux-musl",
 565             "x86_64-unknown-netbsd",
 566         ]
 567         return opt == "true" \
 568             or (opt == "if-available" and self.build in supported_platforms)
 569
 570     def _download_component_helper(
 571         self, filename, pattern, tarball_suffix, stage0=True, key=None
 572     ):
 573         if key is None:
 574             if stage0:
 575                 key = self.stage0_compiler.date
 576             else:
 577                 key = self.rustc_commit
 578         cache_dst = os.path.join(self.build_dir, "cache")
 579         rustc_cache = os.path.join(cache_dst, key)
 580         if not os.path.exists(rustc_cache):
 581             os.makedirs(rustc_cache)
 582
 583         if stage0:
 584             base = self._download_url
 585             url = "dist/{}".format(key)
 586         else:
 587             base = "https://ci-artifacts.rust-lang.org"
 588             url = "rustc-builds/{}".format(self.rustc_commit)
 589         tarball = os.path.join(rustc_cache, filename)
 590         if not os.path.exists(tarball):
 591             get(
 592                 base,
 593                 "{}/{}".format(url, filename),
 594                 tarball,
 595                 self.checksums_sha256,
 596                 verbose=self.verbose,
 597                 do_verify=stage0,
 598             )
 599         unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
 600
 601     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 602         if not llvm_sha:
 603             print("error: could not find commit hash for downloading LLVM")
 604             print("help: maybe your repository history is too shallow?")
 605             print("help: consider disabling `download-ci-llvm`")
 606             print("help: or fetch enough history to include one upstream commit")
 607             exit(1)
 608         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 609         cache_dst = os.path.join(self.build_dir, "cache")
 610         rustc_cache = os.path.join(cache_dst, cache_prefix)
 611         if not os.path.exists(rustc_cache):
 612             os.makedirs(rustc_cache)
 613
 614         base = "https://ci-artifacts.rust-lang.org"
 615         url = "rustc-builds/{}".format(llvm_sha)
 616         if llvm_assertions:
 617             url = url.replace('rustc-builds', 'rustc-builds-alt')
 618         # ci-artifacts are only stored as .xz, not .gz
 619         if not support_xz():
 620             print("error: XZ support is required to download LLVM")
 621             print("help: consider disabling `download-ci-llvm` or using python3")
 622             exit(1)
 623         tarball_suffix = '.tar.xz'
 624         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 625         tarball = os.path.join(rustc_cache, filename)
 626         if not os.path.exists(tarball):
 627             get(
 628                 base,
 629                 "{}/{}".format(url, filename),
 630                 tarball,
 631                 self.checksums_sha256,
 632                 verbose=self.verbose,
 633                 do_verify=False,
 634             )
 635         unpack(tarball, tarball_suffix, self.llvm_root(),
 636                 match="rust-dev",
 637                 verbose=self.verbose)
 638
 639     def fix_bin_or_dylib(self, fname):
 640         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 641         or the RPATH section, to fix the dynamic library search path
 642
 643         This method is only required on NixOS and uses the PatchELF utility to
 644         change the interpreter/RPATH of ELF executables.
 645
 646         Please see https://nixos.org/patchelf.html for more information
 647         """
 648         default_encoding = sys.getdefaultencoding()
 649         try:
 650             ostype = subprocess.check_output(
 651                 ['uname', '-s']).strip().decode(default_encoding)
 652         except subprocess.CalledProcessError:
 653             return
 654         except OSError as reason:
 655             if getattr(reason, 'winerror', None) is not None:
 656                 return
 657             raise reason
 658
 659         if ostype != "Linux":
 660             return
 661
 662         # If the user has asked binaries to be patched for Nix, then
 663         # don't check for NixOS or `/lib`, just continue to the patching.
 664         if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
 665             # Use `/etc/os-release` instead of `/etc/NIXOS`.
 666             # The latter one does not exist on NixOS when using tmpfs as root.
 667             try:
 668                 with open("/etc/os-release", "r") as f:
 669                     if not any(line.strip() == "ID=nixos" for line in f):
 670                         return
 671             except FileNotFoundError:
 672                 return
 673             if os.path.exists("/lib"):
 674                 return
 675
 676         # At this point we're pretty sure the user is running NixOS or
 677         # using Nix
 678         nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
 679         print(nix_os_msg, fname)
 680
 681         # Only build `.nix-deps` once.
 682         nix_deps_dir = self.nix_deps_dir
 683         if not nix_deps_dir:
 684             # Run `nix-build` to "build" each dependency (which will likely reuse
 685             # the existing `/nix/store` copy, or at most download a pre-built copy).
 686             #
 687             # Importantly, we create a gc-root called `.nix-deps` in the `build/`
 688             # directory, but still reference the actual `/nix/store` path in the rpath
 689             # as it makes it significantly more robust against changes to the location of
 690             # the `.nix-deps` location.
 691             #
 692             # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 693             # zlib: Needed as a system dependency of `libLLVM-*.so`.
 694             # patchelf: Needed for patching ELF binaries (see doc comment above).
 695             nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
 696             nix_expr = '''
 697             with (import <nixpkgs> {});
 698             symlinkJoin {
 699               name = "rust-stage0-dependencies";
 700               paths = [
 701                 zlib
 702                 patchelf
 703                 stdenv.cc.bintools
 704               ];
 705             }
 706             '''
 707             try:
 708                 subprocess.check_output([
 709                     "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
 710                 ])
 711             except subprocess.CalledProcessError as reason:
 712                 print("warning: failed to call nix-build:", reason)
 713                 return
 714             self.nix_deps_dir = nix_deps_dir
 715
 716         patchelf = "{}/bin/patchelf".format(nix_deps_dir)
 717         rpath_entries = [
 718             # Relative default, all binary and dynamic libraries we ship
 719             # appear to have this (even when `../lib` is redundant).
 720             "$ORIGIN/../lib",
 721             os.path.join(os.path.realpath(nix_deps_dir), "lib")
 722         ]
 723         patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 724         if not fname.endswith(".so"):
 725             # Finally, set the corret .interp for binaries
 726             with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
 727                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 728
 729         try:
 730             subprocess.check_output([patchelf] + patchelf_args + [fname])
 731         except subprocess.CalledProcessError as reason:
 732             print("warning: failed to call patchelf:", reason)
 733             return
 734
 735     # If `download-rustc` is set, download the most recent commit with CI artifacts
 736     def maybe_download_ci_toolchain(self):
 737         # If `download-rustc` is not set, default to rebuilding.
 738         download_rustc = self.get_toml("download-rustc", section="rust")
 739         if download_rustc is None or download_rustc == "false":
 740             return None
 741         assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
 742
 743         # Handle running from a directory other than the top level
 744         rev_parse = ["git", "rev-parse", "--show-toplevel"]
 745         top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
 746         compiler = "{}/compiler/".format(top_level)
 747         library = "{}/library/".format(top_level)
 748
 749         # Look for a version to compare to based on the current commit.
 750         # Only commits merged by bors will have CI artifacts.
 751         merge_base = [
 752             "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
 753             "--first-parent", "HEAD"
 754         ]
 755         commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
 756         if not commit:
 757             print("error: could not find commit hash for downloading rustc")
 758             print("help: maybe your repository history is too shallow?")
 759             print("help: consider disabling `download-rustc`")
 760             print("help: or fetch enough history to include one upstream commit")
 761             exit(1)
 762
 763         # Warn if there were changes to the compiler or standard library since the ancestor commit.
 764         status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
 765         if status != 0:
 766             if download_rustc == "if-unchanged":
 767                 if self.verbose:
 768                     print("warning: saw changes to compiler/ or library/ since {}; " \
 769                           "ignoring `download-rustc`".format(commit))
 770                 return None
 771             print("warning: `download-rustc` is enabled, but there are changes to " \
 772                   "compiler/ or library/")
 773
 774         if self.verbose:
 775             print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
 776         self.rustc_commit = commit
 777         # FIXME: support downloading artifacts from the beta channel
 778         self.download_toolchain(False, "nightly")
 779
 780     def rustc_stamp(self, stage0):
 781         """Return the path for .rustc-stamp at the given stage
 782
 783         >>> rb = RustBuild()
 784         >>> rb.build_dir = "build"
 785         >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
 786         True
 787         >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
 788         True
 789         """
 790         return os.path.join(self.bin_root(stage0), '.rustc-stamp')
 791
 792     def rustfmt_stamp(self):
 793         """Return the path for .rustfmt-stamp
 794
 795         >>> rb = RustBuild()
 796         >>> rb.build_dir = "build"
 797         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 798         True
 799         """
 800         return os.path.join(self.bin_root(True), '.rustfmt-stamp')
 801
 802     def llvm_stamp(self):
 803         """Return the path for .rustfmt-stamp
 804
 805         >>> rb = RustBuild()
 806         >>> rb.build_dir = "build"
 807         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 808         True
 809         """
 810         return os.path.join(self.llvm_root(), '.llvm-stamp')
 811
 812
 813     def program_out_of_date(self, stamp_path, key):
 814         """Check if the given program stamp is out of date"""
 815         if not os.path.exists(stamp_path) or self.clean:
 816             return True
 817         with open(stamp_path, 'r') as stamp:
 818             return key != stamp.read()
 819
 820     def bin_root(self, stage0):
 821         """Return the binary root directory for the given stage
 822
 823         >>> rb = RustBuild()
 824         >>> rb.build_dir = "build"
 825         >>> rb.bin_root(True) == os.path.join("build", "stage0")
 826         True
 827         >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
 828         True
 829
 830         When the 'build' property is given should be a nested directory:
 831
 832         >>> rb.build = "devel"
 833         >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
 834         True
 835         """
 836         if stage0:
 837             subdir = "stage0"
 838         else:
 839             subdir = "ci-rustc"
 840         return os.path.join(self.build_dir, self.build, subdir)
 841
 842     def llvm_root(self):
 843         """Return the CI LLVM root directory
 844
 845         >>> rb = RustBuild()
 846         >>> rb.build_dir = "build"
 847         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 848         True
 849
 850         When the 'build' property is given should be a nested directory:
 851
 852         >>> rb.build = "devel"
 853         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 854         True
 855         """
 856         return os.path.join(self.build_dir, self.build, "ci-llvm")
 857
 858     def get_toml(self, key, section=None):
 859         """Returns the value of the given key in config.toml, otherwise returns None
 860
 861         >>> rb = RustBuild()
 862         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 863         >>> rb.get_toml("key2")
 864         'value2'
 865
 866         If the key does not exist, the result is None:
 867
 868         >>> rb.get_toml("key3") is None
 869         True
 870
 871         Optionally also matches the section the key appears in
 872
 873         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 874         >>> rb.get_toml('key', 'a')
 875         'value1'
 876         >>> rb.get_toml('key', 'b')
 877         'value2'
 878         >>> rb.get_toml('key', 'c') is None
 879         True
 880
 881         >>> rb.config_toml = 'key1 = true'
 882         >>> rb.get_toml("key1")
 883         'true'
 884         """
 885
 886         cur_section = None
 887         for line in self.config_toml.splitlines():
 888             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 889             if section_match is not None:
 890                 cur_section = section_match.group(1)
 891
 892             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 893             if match is not None:
 894                 value = match.group(1)
 895                 if section is None or section == cur_section:
 896                     return self.get_string(value) or value.strip()
 897         return None
 898
 899     def cargo(self):
 900         """Return config path for cargo"""
 901         return self.program_config('cargo')
 902
 903     def rustc(self, stage0):
 904         """Return config path for rustc"""
 905         return self.program_config('rustc', stage0)
 906
 907     def rustfmt(self):
 908         """Return config path for rustfmt"""
 909         if self.stage0_rustfmt is None:
 910             return None
 911         return self.program_config('rustfmt')
 912
 913     def program_config(self, program, stage0=True):
 914         """Return config path for the given program at the given stage
 915
 916         >>> rb = RustBuild()
 917         >>> rb.config_toml = 'rustc = "rustc"\\n'
 918         >>> rb.program_config('rustc')
 919         'rustc'
 920         >>> rb.config_toml = ''
 921         >>> cargo_path = rb.program_config('cargo', True)
 922         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
 923         ... "bin", "cargo")
 924         True
 925         >>> cargo_path = rb.program_config('cargo', False)
 926         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
 927         ... "bin", "cargo")
 928         True
 929         """
 930         config = self.get_toml(program)
 931         if config:
 932             return os.path.expanduser(config)
 933         return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
 934             program, self.exe_suffix()))
 935
 936     @staticmethod
 937     def get_string(line):
 938         """Return the value between double quotes
 939
 940         >>> RustBuild.get_string('    "devel"   ')
 941         'devel'
 942         >>> RustBuild.get_string("    'devel'   ")
 943         'devel'
 944         >>> RustBuild.get_string('devel') is None
 945         True
 946         >>> RustBuild.get_string('    "devel   ')
 947         ''
 948         """
 949         start = line.find('"')
 950         if start != -1:
 951             end = start + 1 + line[start + 1:].find('"')
 952             return line[start + 1:end]
 953         start = line.find('\'')
 954         if start != -1:
 955             end = start + 1 + line[start + 1:].find('\'')
 956             return line[start + 1:end]
 957         return None
 958
 959     @staticmethod
 960     def exe_suffix():
 961         """Return a suffix for executables"""
 962         if sys.platform == 'win32':
 963             return '.exe'
 964         return ''
 965
 966     def bootstrap_binary(self):
 967         """Return the path of the bootstrap binary
 968
 969         >>> rb = RustBuild()
 970         >>> rb.build_dir = "build"
 971         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 972         ... "debug", "bootstrap")
 973         True
 974         """
 975         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 976
 977     def build_bootstrap(self):
 978         """Build bootstrap"""
 979         print("Building rustbuild")
 980         build_dir = os.path.join(self.build_dir, "bootstrap")
 981         if self.clean and os.path.exists(build_dir):
 982             shutil.rmtree(build_dir)
 983         env = os.environ.copy()
 984         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 985         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 986         if "CARGO_BUILD_TARGET" in env:
 987             del env["CARGO_BUILD_TARGET"]
 988         env["CARGO_TARGET_DIR"] = build_dir
 989         env["RUSTC"] = self.rustc(True)
 990         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 991             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 992             if "LD_LIBRARY_PATH" in env else ""
 993         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 994             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 995             if "DYLD_LIBRARY_PATH" in env else ""
 996         env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
 997             (os.pathsep + env["LIBRARY_PATH"]) \
 998             if "LIBRARY_PATH" in env else ""
 999
1000         # preserve existing RUSTFLAGS
1001         env.setdefault("RUSTFLAGS", "")
1002         build_section = "target.{}".format(self.build)
1003         target_features = []
1004         if self.get_toml("crt-static", build_section) == "true":
1005             target_features += ["+crt-static"]
1006         elif self.get_toml("crt-static", build_section) == "false":
1007             target_features += ["-crt-static"]
1008         if target_features:
1009             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1010         target_linker = self.get_toml("linker", build_section)
1011         if target_linker is not None:
1012             env["RUSTFLAGS"] += " -C linker=" + target_linker
1013         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1014         env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1015         if self.get_toml("deny-warnings", "rust") != "false":
1016             env["RUSTFLAGS"] += " -Dwarnings"
1017
1018         env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1019             os.pathsep + env["PATH"]
1020         if not os.path.isfile(self.cargo()):
1021             raise Exception("no cargo executable found at `{}`".format(
1022                 self.cargo()))
1023         args = [self.cargo(), "build", "--manifest-path",
1024                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1025         for _ in range(0, self.verbose):
1026             args.append("--verbose")
1027         if self.use_locked_deps:
1028             args.append("--locked")
1029         if self.use_vendored_sources:
1030             args.append("--frozen")
1031         run(args, env=env, verbose=self.verbose)
1032
1033     def build_triple(self):
1034         """Build triple as in LLVM
1035
1036         Note that `default_build_triple` is moderately expensive,
1037         so use `self.build` where possible.
1038         """
1039         config = self.get_toml('build')
1040         if config:
1041             return config
1042         return default_build_triple(self.verbose)
1043
1044     def check_submodule(self, module, slow_submodules):
1045         if not slow_submodules:
1046             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1047                                            cwd=os.path.join(self.rust_root, module),
1048                                            stdout=subprocess.PIPE)
1049             return checked_out
1050         else:
1051             return None
1052
1053     def update_submodule(self, module, checked_out, recorded_submodules):
1054         module_path = os.path.join(self.rust_root, module)
1055
1056         if checked_out is not None:
1057             default_encoding = sys.getdefaultencoding()
1058             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1059             if recorded_submodules[module] == checked_out:
1060                 return
1061
1062         print("Updating submodule", module)
1063
1064         run(["git", "submodule", "-q", "sync", module],
1065             cwd=self.rust_root, verbose=self.verbose)
1066
1067         update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1068         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1069             update_args.append("--progress")
1070         update_args.append(module)
1071         try:
1072             run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1073         except RuntimeError:
1074             print("Failed updating submodule. This is probably due to uncommitted local changes.")
1075             print('Either stash the changes by running "git stash" within the submodule\'s')
1076             print('directory, reset them by running "git reset --hard", or commit them.')
1077             print("To reset all submodules' changes run", end=" ")
1078             print('"git submodule foreach --recursive git reset --hard".')
1079             raise SystemExit(1)
1080
1081         run(["git", "reset", "-q", "--hard"],
1082             cwd=module_path, verbose=self.verbose)
1083         run(["git", "clean", "-qdfx"],
1084             cwd=module_path, verbose=self.verbose)
1085
1086     def update_submodules(self):
1087         """Update submodules"""
1088         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1089                 self.get_toml('submodules') == "false":
1090             return
1091
1092         default_encoding = sys.getdefaultencoding()
1093
1094         # check the existence and version of 'git' command
1095         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1096         self.git_version = distutils.version.LooseVersion(git_version_str)
1097
1098         slow_submodules = self.get_toml('fast-submodules') == "false"
1099         start_time = time()
1100         if slow_submodules:
1101             print('Unconditionally updating submodules')
1102         else:
1103             print('Updating only changed submodules')
1104         default_encoding = sys.getdefaultencoding()
1105         # Only update submodules that are needed to build bootstrap.  These are needed because Cargo
1106         # currently requires everything in a workspace to be "locally present" when starting a
1107         # build, and will give a hard error if any Cargo.toml files are missing.
1108         # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1109         #   share a workspace with any tools - maybe it could be excluded from the workspace?
1110         #   That will still require cloning the submodules the second you check the standard
1111         #   library, though...
1112         # FIXME: Is there a way to avoid hard-coding the submodules required?
1113         # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1114         submodules = [
1115             "src/tools/rust-installer",
1116             "src/tools/cargo",
1117             "src/tools/rls",
1118             "src/tools/miri",
1119             "library/backtrace",
1120             "library/stdarch"
1121         ]
1122         filtered_submodules = []
1123         submodules_names = []
1124         for module in submodules:
1125             check = self.check_submodule(module, slow_submodules)
1126             filtered_submodules.append((module, check))
1127             submodules_names.append(module)
1128         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1129                                     cwd=self.rust_root, stdout=subprocess.PIPE)
1130         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1131         # { filename: hash }
1132         recorded_submodules = {}
1133         for data in recorded:
1134             # [mode, kind, hash, filename]
1135             data = data.split()
1136             recorded_submodules[data[3]] = data[2]
1137         for module in filtered_submodules:
1138             self.update_submodule(module[0], module[1], recorded_submodules)
1139         print("  Submodules updated in %.2f seconds" % (time() - start_time))
1140
1141     def set_dist_environment(self, url):
1142         """Set download URL for normal environment"""
1143         if 'RUSTUP_DIST_SERVER' in os.environ:
1144             self._download_url = os.environ['RUSTUP_DIST_SERVER']
1145         else:
1146             self._download_url = url
1147
1148     def check_vendored_status(self):
1149         """Check that vendoring is configured properly"""
1150         vendor_dir = os.path.join(self.rust_root, 'vendor')
1151         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1152             if os.environ.get('USER') != os.environ['SUDO_USER']:
1153                 self.use_vendored_sources = True
1154                 print('info: looks like you are running this command under `sudo`')
1155                 print('      and so in order to preserve your $HOME this will now')
1156                 print('      use vendored sources by default.')
1157                 if not os.path.exists(vendor_dir):
1158                     print('error: vendoring required, but vendor directory does not exist.')
1159                     print('       Run `cargo vendor` without sudo to initialize the '
1160                           'vendor directory.')
1161                     raise Exception("{} not found".format(vendor_dir))
1162
1163         if self.use_vendored_sources:
1164             config = ("[source.crates-io]\n"
1165                       "replace-with = 'vendored-sources'\n"
1166                       "registry = 'https://example.com'\n"
1167                       "\n"
1168                       "[source.vendored-sources]\n"
1169                       "directory = '{}/vendor'\n"
1170                       .format(self.rust_root))
1171             if not os.path.exists('.cargo'):
1172                 os.makedirs('.cargo')
1173                 with output('.cargo/config') as cargo_config:
1174                     cargo_config.write(config)
1175             else:
1176                 print('info: using vendored source, but .cargo/config is already present.')
1177                 print('      Reusing the current configuration file. But you may want to '
1178                       'configure vendoring like this:')
1179                 print(config)
1180         else:
1181             if os.path.exists('.cargo'):
1182                 shutil.rmtree('.cargo')
1183
1184     def ensure_vendored(self):
1185         """Ensure that the vendored sources are available if needed"""
1186         vendor_dir = os.path.join(self.rust_root, 'vendor')
1187         # Note that this does not handle updating the vendored dependencies if
1188         # the rust git repository is updated. Normal development usually does
1189         # not use vendoring, so hopefully this isn't too much of a problem.
1190         if self.use_vendored_sources and not os.path.exists(vendor_dir):
1191             run([
1192                 self.cargo(),
1193                 "vendor",
1194                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1195                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1196             ], verbose=self.verbose, cwd=self.rust_root)
1197
1198
1199 def bootstrap(help_triggered):
1200     """Configure, fetch, build and run the initial bootstrap"""
1201
1202     # If the user is asking for help, let them know that the whole download-and-build
1203     # process has to happen before anything is printed out.
1204     if help_triggered:
1205         print("info: Downloading and building bootstrap before processing --help")
1206         print("      command. See src/bootstrap/README.md for help with common")
1207         print("      commands.")
1208
1209     parser = argparse.ArgumentParser(description='Build rust')
1210     parser.add_argument('--config')
1211     parser.add_argument('--build')
1212     parser.add_argument('--clean', action='store_true')
1213     parser.add_argument('-v', '--verbose', action='count', default=0)
1214
1215     args = [a for a in sys.argv if a != '-h' and a != '--help']
1216     args, _ = parser.parse_known_args(args)
1217
1218     # Configure initial bootstrap
1219     build = RustBuild()
1220     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1221     build.verbose = args.verbose
1222     build.clean = args.clean
1223
1224     # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then fallback to `config.toml` (if it
1225     # exists).
1226     toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1227     if not toml_path and os.path.exists('config.toml'):
1228         toml_path = 'config.toml'
1229
1230     if toml_path:
1231         if not os.path.exists(toml_path):
1232             toml_path = os.path.join(build.rust_root, toml_path)
1233
1234         with open(toml_path) as config:
1235             build.config_toml = config.read()
1236
1237     profile = build.get_toml('profile')
1238     if profile is not None:
1239         include_file = 'config.{}.toml'.format(profile)
1240         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1241         include_path = os.path.join(include_dir, include_file)
1242         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1243         # specific key, so appending our defaults at the end allows the user to override them
1244         with open(include_path) as included_toml:
1245             build.config_toml += os.linesep + included_toml.read()
1246
1247     config_verbose = build.get_toml('verbose', 'build')
1248     if config_verbose is not None:
1249         build.verbose = max(build.verbose, int(config_verbose))
1250
1251     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1252
1253     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1254
1255     build.check_vendored_status()
1256
1257     build_dir = build.get_toml('build-dir', 'build') or 'build'
1258     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1259
1260     with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1261         data = json.load(f)
1262     build.checksums_sha256 = data["checksums_sha256"]
1263     build.stage0_compiler = Stage0Toolchain(data["compiler"])
1264     if data.get("rustfmt") is not None:
1265         build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1266
1267     build.set_dist_environment(data["dist_server"])
1268
1269     build.build = args.build or build.build_triple()
1270
1271     # Acquire the lock before doing any build actions
1272     # The lock is released when `lock` is dropped
1273     if not os.path.exists(build.build_dir):
1274         os.makedirs(build.build_dir)
1275     lock = acquire_lock(build.build_dir)
1276     build.update_submodules()
1277
1278     # Fetch/build the bootstrap
1279     build.download_toolchain()
1280     # Download the master compiler if `download-rustc` is set
1281     build.maybe_download_ci_toolchain()
1282     sys.stdout.flush()
1283     build.ensure_vendored()
1284     build.build_bootstrap()
1285     sys.stdout.flush()
1286
1287     # Run the bootstrap
1288     args = [build.bootstrap_binary()]
1289     args.extend(sys.argv[1:])
1290     env = os.environ.copy()
1291     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1292     env["BOOTSTRAP_PYTHON"] = sys.executable
1293     env["BUILD_DIR"] = build.build_dir
1294     env["RUSTC_BOOTSTRAP"] = '1'
1295     if toml_path:
1296         env["BOOTSTRAP_CONFIG"] = toml_path
1297     if build.rustc_commit is not None:
1298         env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1299     run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1300
1301
1302 def main():
1303     """Entry point for the bootstrap process"""
1304     start_time = time()
1305
1306     # x.py help <cmd> ...
1307     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1308         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1309
1310     help_triggered = (
1311         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1312     try:
1313         bootstrap(help_triggered)
1314         if not help_triggered:
1315             print("Build completed successfully in {}".format(
1316                 format_build_time(time() - start_time)))
1317     except (SystemExit, KeyboardInterrupt) as error:
1318         if hasattr(error, 'code') and isinstance(error.code, int):
1319             exit_code = error.code
1320         else:
1321             exit_code = 1
1322             print(error)
1323         if not help_triggered:
1324             print("Build completed unsuccessfully in {}".format(
1325                 format_build_time(time() - start_time)))
1326         sys.exit(exit_code)
1327
1328
1329 if __name__ == '__main__':
1330     main()