src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple(verbose):
 191     """Build triple as in LLVM"""
 192     # If the user already has a host build triple with an existing `rustc`
 193     # install, use their preference. This fixes most issues with Windows builds
 194     # being detected as GNU instead of MSVC.
 195     default_encoding = sys.getdefaultencoding()
 196     try:
 197         version = subprocess.check_output(["rustc", "--version", "--verbose"])
 198         version = version.decode(default_encoding)
 199         host = next(x for x in version.split('\n') if x.startswith("host: "))
 200         triple = host.split("host: ")[1]
 201         if verbose:
 202             print("detected default triple {}".format(triple))
 203         return triple
 204     except Exception as e:
 205         if verbose:
 206             print("rustup not detected: {}".format(e))
 207             print("falling back to auto-detect")
 208
 209     required = sys.platform != 'win32'
 210     ostype = require(["uname", "-s"], exit=required)
 211     cputype = require(['uname', '-m'], exit=required)
 212
 213     # If we do not have `uname`, assume Windows.
 214     if ostype is None or cputype is None:
 215         return 'x86_64-pc-windows-msvc'
 216
 217     ostype = ostype.decode(default_encoding)
 218     cputype = cputype.decode(default_encoding)
 219
 220     # The goal here is to come up with the same triple as LLVM would,
 221     # at least for the subset of platforms we're willing to target.
 222     ostype_mapper = {
 223         'Darwin': 'apple-darwin',
 224         'DragonFly': 'unknown-dragonfly',
 225         'FreeBSD': 'unknown-freebsd',
 226         'Haiku': 'unknown-haiku',
 227         'NetBSD': 'unknown-netbsd',
 228         'OpenBSD': 'unknown-openbsd'
 229     }
 230
 231     # Consider the direct transformation first and then the special cases
 232     if ostype in ostype_mapper:
 233         ostype = ostype_mapper[ostype]
 234     elif ostype == 'Linux':
 235         os_from_sp = subprocess.check_output(
 236             ['uname', '-o']).strip().decode(default_encoding)
 237         if os_from_sp == 'Android':
 238             ostype = 'linux-android'
 239         else:
 240             ostype = 'unknown-linux-gnu'
 241     elif ostype == 'SunOS':
 242         ostype = 'sun-solaris'
 243         # On Solaris, uname -m will return a machine classification instead
 244         # of a cpu type, so uname -p is recommended instead.  However, the
 245         # output from that option is too generic for our purposes (it will
 246         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 247         # must be used instead.
 248         cputype = require(['isainfo', '-k']).decode(default_encoding)
 249     elif ostype.startswith('MINGW'):
 250         # msys' `uname` does not print gcc configuration, but prints msys
 251         # configuration. so we cannot believe `uname -m`:
 252         # msys1 is always i686 and msys2 is always x86_64.
 253         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 254         # MINGW64 on x86_64.
 255         ostype = 'pc-windows-gnu'
 256         cputype = 'i686'
 257         if os.environ.get('MSYSTEM') == 'MINGW64':
 258             cputype = 'x86_64'
 259     elif ostype.startswith('MSYS'):
 260         ostype = 'pc-windows-gnu'
 261     elif ostype.startswith('CYGWIN_NT'):
 262         cputype = 'i686'
 263         if ostype.endswith('WOW64'):
 264             cputype = 'x86_64'
 265         ostype = 'pc-windows-gnu'
 266     elif sys.platform == 'win32':
 267         # Some Windows platforms might have a `uname` command that returns a
 268         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 269         # these cases, fall back to using sys.platform.
 270         return 'x86_64-pc-windows-msvc'
 271     else:
 272         err = "unknown OS type: {}".format(ostype)
 273         sys.exit(err)
 274
 275     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 276         cputype = subprocess.check_output(
 277               ['uname', '-p']).strip().decode(default_encoding)
 278     cputype_mapper = {
 279         'BePC': 'i686',
 280         'aarch64': 'aarch64',
 281         'amd64': 'x86_64',
 282         'arm64': 'aarch64',
 283         'i386': 'i686',
 284         'i486': 'i686',
 285         'i686': 'i686',
 286         'i786': 'i686',
 287         'powerpc': 'powerpc',
 288         'powerpc64': 'powerpc64',
 289         'powerpc64le': 'powerpc64le',
 290         'ppc': 'powerpc',
 291         'ppc64': 'powerpc64',
 292         'ppc64le': 'powerpc64le',
 293         's390x': 's390x',
 294         'x64': 'x86_64',
 295         'x86': 'i686',
 296         'x86-64': 'x86_64',
 297         'x86_64': 'x86_64'
 298     }
 299
 300     # Consider the direct transformation first and then the special cases
 301     if cputype in cputype_mapper:
 302         cputype = cputype_mapper[cputype]
 303     elif cputype in {'xscale', 'arm'}:
 304         cputype = 'arm'
 305         if ostype == 'linux-android':
 306             ostype = 'linux-androideabi'
 307         elif ostype == 'unknown-freebsd':
 308             cputype = subprocess.check_output(
 309                 ['uname', '-p']).strip().decode(default_encoding)
 310             ostype = 'unknown-freebsd'
 311     elif cputype == 'armv6l':
 312         cputype = 'arm'
 313         if ostype == 'linux-android':
 314             ostype = 'linux-androideabi'
 315         else:
 316             ostype += 'eabihf'
 317     elif cputype in {'armv7l', 'armv8l'}:
 318         cputype = 'armv7'
 319         if ostype == 'linux-android':
 320             ostype = 'linux-androideabi'
 321         else:
 322             ostype += 'eabihf'
 323     elif cputype == 'mips':
 324         if sys.byteorder == 'big':
 325             cputype = 'mips'
 326         elif sys.byteorder == 'little':
 327             cputype = 'mipsel'
 328         else:
 329             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 330     elif cputype == 'mips64':
 331         if sys.byteorder == 'big':
 332             cputype = 'mips64'
 333         elif sys.byteorder == 'little':
 334             cputype = 'mips64el'
 335         else:
 336             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 337         # only the n64 ABI is supported, indicate it
 338         ostype += 'abi64'
 339     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 340         pass
 341     else:
 342         err = "unknown cpu type: {}".format(cputype)
 343         sys.exit(err)
 344
 345     return "{}-{}".format(cputype, ostype)
 346
 347
 348 @contextlib.contextmanager
 349 def output(filepath):
 350     tmp = filepath + '.tmp'
 351     with open(tmp, 'w') as f:
 352         yield f
 353     try:
 354         if os.path.exists(filepath):
 355             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 356     except OSError:
 357         shutil.copy2(tmp, filepath)
 358         os.remove(tmp)
 359         return
 360     os.rename(tmp, filepath)
 361
 362
 363 class RustBuild(object):
 364     """Provide all the methods required to build Rust"""
 365     def __init__(self):
 366         self.date = ''
 367         self._download_url = ''
 368         self.rustc_channel = ''
 369         self.rustfmt_channel = ''
 370         self.build = ''
 371         self.build_dir = ''
 372         self.clean = False
 373         self.config_toml = ''
 374         self.rust_root = ''
 375         self.use_locked_deps = ''
 376         self.use_vendored_sources = ''
 377         self.verbose = False
 378         self.git_version = None
 379         self.nix_deps_dir = None
 380
 381     def download_stage0(self):
 382         """Fetch the build system for Rust, written in Rust
 383
 384         This method will build a cache directory, then it will fetch the
 385         tarball which has the stage0 compiler used to then bootstrap the Rust
 386         compiler itself.
 387
 388         Each downloaded tarball is extracted, after that, the script
 389         will move all the content to the right place.
 390         """
 391         rustc_channel = self.rustc_channel
 392         rustfmt_channel = self.rustfmt_channel
 393
 394         if self.rustc().startswith(self.bin_root()) and \
 395                 (not os.path.exists(self.rustc()) or
 396                  self.program_out_of_date(self.rustc_stamp(), self.date)):
 397             if os.path.exists(self.bin_root()):
 398                 shutil.rmtree(self.bin_root())
 399             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 400             filename = "rust-std-{}-{}{}".format(
 401                 rustc_channel, self.build, tarball_suffix)
 402             pattern = "rust-std-{}".format(self.build)
 403             self._download_stage0_helper(filename, pattern, tarball_suffix)
 404             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 405                                               tarball_suffix)
 406             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 407             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 408                                               tarball_suffix)
 409             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 410             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 411             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 412             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 413             lib_dir = "{}/lib".format(self.bin_root())
 414             for lib in os.listdir(lib_dir):
 415                 if lib.endswith(".so"):
 416                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 417             with output(self.rustc_stamp()) as rust_stamp:
 418                 rust_stamp.write(self.date)
 419
 420         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 421             not os.path.exists(self.rustfmt())
 422             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 423         ):
 424             if rustfmt_channel:
 425                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 426                 [channel, date] = rustfmt_channel.split('-', 1)
 427                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 428                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 429                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 430                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 431                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 432                     rustfmt_stamp.write(self.rustfmt_channel)
 433
 434         if self.downloading_llvm():
 435             # We want the most recent LLVM submodule update to avoid downloading
 436             # LLVM more often than necessary.
 437             #
 438             # This git command finds that commit SHA, looking for bors-authored
 439             # merges that modified src/llvm-project.
 440             #
 441             # This works even in a repository that has not yet initialized
 442             # submodules.
 443             top_level = subprocess.check_output([
 444                 "git", "rev-parse", "--show-toplevel",
 445             ]).decode(sys.getdefaultencoding()).strip()
 446             llvm_sha = subprocess.check_output([
 447                 "git", "log", "--author=bors", "--format=%H", "-n1",
 448                 "-m", "--first-parent",
 449                 "--",
 450                 "{}/src/llvm-project".format(top_level),
 451                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 452             ]).decode(sys.getdefaultencoding()).strip()
 453             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 454             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 455                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 456                 for binary in ["llvm-config", "FileCheck"]:
 457                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 458                 with output(self.llvm_stamp()) as llvm_stamp:
 459                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 460
 461     def downloading_llvm(self):
 462         opt = self.get_toml('download-ci-llvm', 'llvm')
 463         return opt == "true" \
 464             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 465
 466     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 467         if date is None:
 468             date = self.date
 469         cache_dst = os.path.join(self.build_dir, "cache")
 470         rustc_cache = os.path.join(cache_dst, date)
 471         if not os.path.exists(rustc_cache):
 472             os.makedirs(rustc_cache)
 473
 474         url = "{}/dist/{}".format(self._download_url, date)
 475         tarball = os.path.join(rustc_cache, filename)
 476         if not os.path.exists(tarball):
 477             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 478         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 479
 480     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 481         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 482         cache_dst = os.path.join(self.build_dir, "cache")
 483         rustc_cache = os.path.join(cache_dst, cache_prefix)
 484         if not os.path.exists(rustc_cache):
 485             os.makedirs(rustc_cache)
 486
 487         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 488         if llvm_assertions:
 489             url = url.replace('rustc-builds', 'rustc-builds-alt')
 490         # ci-artifacts are only stored as .xz, not .gz
 491         if not support_xz():
 492             print("error: XZ support is required to download LLVM")
 493             print("help: consider disabling `download-ci-llvm` or using python3")
 494             exit(1)
 495         tarball_suffix = '.tar.xz'
 496         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 497         tarball = os.path.join(rustc_cache, filename)
 498         if not os.path.exists(tarball):
 499             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 500         unpack(tarball, tarball_suffix, self.llvm_root(),
 501                 match="rust-dev",
 502                 verbose=self.verbose)
 503
 504     def fix_bin_or_dylib(self, fname):
 505         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 506         or the RPATH section, to fix the dynamic library search path
 507
 508         This method is only required on NixOS and uses the PatchELF utility to
 509         change the interpreter/RPATH of ELF executables.
 510
 511         Please see https://nixos.org/patchelf.html for more information
 512         """
 513         default_encoding = sys.getdefaultencoding()
 514         try:
 515             ostype = subprocess.check_output(
 516                 ['uname', '-s']).strip().decode(default_encoding)
 517         except subprocess.CalledProcessError:
 518             return
 519         except OSError as reason:
 520             if getattr(reason, 'winerror', None) is not None:
 521                 return
 522             raise reason
 523
 524         if ostype != "Linux":
 525             return
 526
 527         if not os.path.exists("/etc/NIXOS"):
 528             return
 529         if os.path.exists("/lib"):
 530             return
 531
 532         # At this point we're pretty sure the user is running NixOS
 533         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 534         print(nix_os_msg, fname)
 535
 536         # Only build `stage0/.nix-deps` once.
 537         nix_deps_dir = self.nix_deps_dir
 538         if not nix_deps_dir:
 539             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 540             if not os.path.exists(nix_deps_dir):
 541                 os.makedirs(nix_deps_dir)
 542
 543             nix_deps = [
 544                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 545                 "stdenv.cc.bintools",
 546
 547                 # Needed as a system dependency of `libLLVM-*.so`.
 548                 "zlib",
 549
 550                 # Needed for patching ELF binaries (see doc comment above).
 551                 "patchelf",
 552             ]
 553
 554             # Run `nix-build` to "build" each dependency (which will likely reuse
 555             # the existing `/nix/store` copy, or at most download a pre-built copy).
 556             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 557             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 558             # ensuring garbage collection will never remove the `/nix/store` path
 559             # (which would break our patched binaries that hardcode those paths).
 560             for dep in nix_deps:
 561                 try:
 562                     subprocess.check_output([
 563                         "nix-build", "<nixpkgs>",
 564                         "-A", dep,
 565                         "-o", "{}/{}".format(nix_deps_dir, dep),
 566                     ])
 567                 except subprocess.CalledProcessError as reason:
 568                     print("warning: failed to call nix-build:", reason)
 569                     return
 570
 571             self.nix_deps_dir = nix_deps_dir
 572
 573         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 574
 575         if fname.endswith(".so"):
 576             # Dynamic library, patch RPATH to point to system dependencies.
 577             dylib_deps = ["zlib"]
 578             rpath_entries = [
 579                 # Relative default, all binary and dynamic libraries we ship
 580                 # appear to have this (even when `../lib` is redundant).
 581                 "$ORIGIN/../lib",
 582             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 583             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 584         else:
 585             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 586             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 587                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 588
 589         try:
 590             subprocess.check_output([patchelf] + patchelf_args + [fname])
 591         except subprocess.CalledProcessError as reason:
 592             print("warning: failed to call patchelf:", reason)
 593             return
 594
 595     def rustc_stamp(self):
 596         """Return the path for .rustc-stamp
 597
 598         >>> rb = RustBuild()
 599         >>> rb.build_dir = "build"
 600         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 601         True
 602         """
 603         return os.path.join(self.bin_root(), '.rustc-stamp')
 604
 605     def rustfmt_stamp(self):
 606         """Return the path for .rustfmt-stamp
 607
 608         >>> rb = RustBuild()
 609         >>> rb.build_dir = "build"
 610         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 611         True
 612         """
 613         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 614
 615     def llvm_stamp(self):
 616         """Return the path for .rustfmt-stamp
 617
 618         >>> rb = RustBuild()
 619         >>> rb.build_dir = "build"
 620         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 621         True
 622         """
 623         return os.path.join(self.llvm_root(), '.llvm-stamp')
 624
 625
 626     def program_out_of_date(self, stamp_path, key):
 627         """Check if the given program stamp is out of date"""
 628         if not os.path.exists(stamp_path) or self.clean:
 629             return True
 630         with open(stamp_path, 'r') as stamp:
 631             return key != stamp.read()
 632
 633     def bin_root(self):
 634         """Return the binary root directory
 635
 636         >>> rb = RustBuild()
 637         >>> rb.build_dir = "build"
 638         >>> rb.bin_root() == os.path.join("build", "stage0")
 639         True
 640
 641         When the 'build' property is given should be a nested directory:
 642
 643         >>> rb.build = "devel"
 644         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 645         True
 646         """
 647         return os.path.join(self.build_dir, self.build, "stage0")
 648
 649     def llvm_root(self):
 650         """Return the CI LLVM root directory
 651
 652         >>> rb = RustBuild()
 653         >>> rb.build_dir = "build"
 654         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 655         True
 656
 657         When the 'build' property is given should be a nested directory:
 658
 659         >>> rb.build = "devel"
 660         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 661         True
 662         """
 663         return os.path.join(self.build_dir, self.build, "ci-llvm")
 664
 665     def get_toml(self, key, section=None):
 666         """Returns the value of the given key in config.toml, otherwise returns None
 667
 668         >>> rb = RustBuild()
 669         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 670         >>> rb.get_toml("key2")
 671         'value2'
 672
 673         If the key does not exists, the result is None:
 674
 675         >>> rb.get_toml("key3") is None
 676         True
 677
 678         Optionally also matches the section the key appears in
 679
 680         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 681         >>> rb.get_toml('key', 'a')
 682         'value1'
 683         >>> rb.get_toml('key', 'b')
 684         'value2'
 685         >>> rb.get_toml('key', 'c') is None
 686         True
 687
 688         >>> rb.config_toml = 'key1 = true'
 689         >>> rb.get_toml("key1")
 690         'true'
 691         """
 692
 693         cur_section = None
 694         for line in self.config_toml.splitlines():
 695             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 696             if section_match is not None:
 697                 cur_section = section_match.group(1)
 698
 699             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 700             if match is not None:
 701                 value = match.group(1)
 702                 if section is None or section == cur_section:
 703                     return self.get_string(value) or value.strip()
 704         return None
 705
 706     def cargo(self):
 707         """Return config path for cargo"""
 708         return self.program_config('cargo')
 709
 710     def rustc(self):
 711         """Return config path for rustc"""
 712         return self.program_config('rustc')
 713
 714     def rustfmt(self):
 715         """Return config path for rustfmt"""
 716         if not self.rustfmt_channel:
 717             return None
 718         return self.program_config('rustfmt')
 719
 720     def program_config(self, program):
 721         """Return config path for the given program
 722
 723         >>> rb = RustBuild()
 724         >>> rb.config_toml = 'rustc = "rustc"\\n'
 725         >>> rb.program_config('rustc')
 726         'rustc'
 727         >>> rb.config_toml = ''
 728         >>> cargo_path = rb.program_config('cargo')
 729         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 730         ... "bin", "cargo")
 731         True
 732         """
 733         config = self.get_toml(program)
 734         if config:
 735             return os.path.expanduser(config)
 736         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 737             program, self.exe_suffix()))
 738
 739     @staticmethod
 740     def get_string(line):
 741         """Return the value between double quotes
 742
 743         >>> RustBuild.get_string('    "devel"   ')
 744         'devel'
 745         >>> RustBuild.get_string("    'devel'   ")
 746         'devel'
 747         >>> RustBuild.get_string('devel') is None
 748         True
 749         >>> RustBuild.get_string('    "devel   ')
 750         ''
 751         """
 752         start = line.find('"')
 753         if start != -1:
 754             end = start + 1 + line[start + 1:].find('"')
 755             return line[start + 1:end]
 756         start = line.find('\'')
 757         if start != -1:
 758             end = start + 1 + line[start + 1:].find('\'')
 759             return line[start + 1:end]
 760         return None
 761
 762     @staticmethod
 763     def exe_suffix():
 764         """Return a suffix for executables"""
 765         if sys.platform == 'win32':
 766             return '.exe'
 767         return ''
 768
 769     def bootstrap_binary(self):
 770         """Return the path of the bootstrap binary
 771
 772         >>> rb = RustBuild()
 773         >>> rb.build_dir = "build"
 774         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 775         ... "debug", "bootstrap")
 776         True
 777         """
 778         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 779
 780     def build_bootstrap(self):
 781         """Build bootstrap"""
 782         build_dir = os.path.join(self.build_dir, "bootstrap")
 783         if self.clean and os.path.exists(build_dir):
 784             shutil.rmtree(build_dir)
 785         env = os.environ.copy()
 786         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 787         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 788         if "CARGO_BUILD_TARGET" in env:
 789             del env["CARGO_BUILD_TARGET"]
 790         env["CARGO_TARGET_DIR"] = build_dir
 791         env["RUSTC"] = self.rustc()
 792         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 793             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 794             if "LD_LIBRARY_PATH" in env else ""
 795         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 796             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 797             if "DYLD_LIBRARY_PATH" in env else ""
 798         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 799             (os.pathsep + env["LIBRARY_PATH"]) \
 800             if "LIBRARY_PATH" in env else ""
 801         # preserve existing RUSTFLAGS
 802         env.setdefault("RUSTFLAGS", "")
 803         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 804
 805         build_section = "target.{}".format(self.build)
 806         target_features = []
 807         if self.get_toml("crt-static", build_section) == "true":
 808             target_features += ["+crt-static"]
 809         elif self.get_toml("crt-static", build_section) == "false":
 810             target_features += ["-crt-static"]
 811         if target_features:
 812             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 813         target_linker = self.get_toml("linker", build_section)
 814         if target_linker is not None:
 815             env["RUSTFLAGS"] += " -C linker=" + target_linker
 816         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 817         if self.get_toml("deny-warnings", "rust") != "false":
 818             env["RUSTFLAGS"] += " -Dwarnings"
 819
 820         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 821             os.pathsep + env["PATH"]
 822         if not os.path.isfile(self.cargo()):
 823             raise Exception("no cargo executable found at `{}`".format(
 824                 self.cargo()))
 825         args = [self.cargo(), "build", "--manifest-path",
 826                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 827         for _ in range(1, self.verbose):
 828             args.append("--verbose")
 829         if self.use_locked_deps:
 830             args.append("--locked")
 831         if self.use_vendored_sources:
 832             args.append("--frozen")
 833         run(args, env=env, verbose=self.verbose)
 834
 835     def build_triple(self):
 836         """Build triple as in LLVM
 837
 838         Note that `default_build_triple` is moderately expensive,
 839         so use `self.build` where possible.
 840         """
 841         config = self.get_toml('build')
 842         if config:
 843             return config
 844         return default_build_triple(self.verbose)
 845
 846     def check_submodule(self, module, slow_submodules):
 847         if not slow_submodules:
 848             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 849                                            cwd=os.path.join(self.rust_root, module),
 850                                            stdout=subprocess.PIPE)
 851             return checked_out
 852         else:
 853             return None
 854
 855     def update_submodule(self, module, checked_out, recorded_submodules):
 856         module_path = os.path.join(self.rust_root, module)
 857
 858         if checked_out is not None:
 859             default_encoding = sys.getdefaultencoding()
 860             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 861             if recorded_submodules[module] == checked_out:
 862                 return
 863
 864         print("Updating submodule", module)
 865
 866         run(["git", "submodule", "-q", "sync", module],
 867             cwd=self.rust_root, verbose=self.verbose)
 868
 869         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 870         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 871             update_args.append("--progress")
 872         update_args.append(module)
 873         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 874
 875         run(["git", "reset", "-q", "--hard"],
 876             cwd=module_path, verbose=self.verbose)
 877         run(["git", "clean", "-qdfx"],
 878             cwd=module_path, verbose=self.verbose)
 879
 880     def update_submodules(self):
 881         """Update submodules"""
 882         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 883                 self.get_toml('submodules') == "false":
 884             return
 885
 886         default_encoding = sys.getdefaultencoding()
 887
 888         # check the existence and version of 'git' command
 889         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 890         self.git_version = distutils.version.LooseVersion(git_version_str)
 891
 892         slow_submodules = self.get_toml('fast-submodules') == "false"
 893         start_time = time()
 894         if slow_submodules:
 895             print('Unconditionally updating all submodules')
 896         else:
 897             print('Updating only changed submodules')
 898         default_encoding = sys.getdefaultencoding()
 899         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 900             ["git", "config", "--file",
 901              os.path.join(self.rust_root, ".gitmodules"),
 902              "--get-regexp", "path"]
 903         ).decode(default_encoding).splitlines()]
 904         filtered_submodules = []
 905         submodules_names = []
 906         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
 907         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
 908         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
 909             or "llvm" in self.get_toml('codegen-backends', 'rust')
 910         for module in submodules:
 911             if module.endswith("llvm-project"):
 912                 # Don't sync the llvm-project submodule if an external LLVM was
 913                 # provided, if we are downloading LLVM or if the LLVM backend is
 914                 # not being built. Also, if the submodule has been initialized
 915                 # already, sync it anyways so that it doesn't mess up contributor
 916                 # pull requests.
 917                 if external_llvm_provided or not llvm_needed:
 918                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
 919                         continue
 920             check = self.check_submodule(module, slow_submodules)
 921             filtered_submodules.append((module, check))
 922             submodules_names.append(module)
 923         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 924                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 925         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 926         recorded_submodules = {}
 927         for data in recorded:
 928             data = data.split()
 929             recorded_submodules[data[3]] = data[2]
 930         for module in filtered_submodules:
 931             self.update_submodule(module[0], module[1], recorded_submodules)
 932         print("Submodules updated in %.2f seconds" % (time() - start_time))
 933
 934     def set_normal_environment(self):
 935         """Set download URL for normal environment"""
 936         if 'RUSTUP_DIST_SERVER' in os.environ:
 937             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 938         else:
 939             self._download_url = 'https://static.rust-lang.org'
 940
 941     def set_dev_environment(self):
 942         """Set download URL for development environment"""
 943         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 944             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 945         else:
 946             self._download_url = 'https://dev-static.rust-lang.org'
 947
 948     def check_vendored_status(self):
 949         """Check that vendoring is configured properly"""
 950         vendor_dir = os.path.join(self.rust_root, 'vendor')
 951         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 952             if os.environ.get('USER') != os.environ['SUDO_USER']:
 953                 self.use_vendored_sources = True
 954                 print('info: looks like you are running this command under `sudo`')
 955                 print('      and so in order to preserve your $HOME this will now')
 956                 print('      use vendored sources by default.')
 957                 if not os.path.exists(vendor_dir):
 958                     print('error: vendoring required, but vendor directory does not exist.')
 959                     print('       Run `cargo vendor` without sudo to initialize the '
 960                           'vendor directory.')
 961                     raise Exception("{} not found".format(vendor_dir))
 962
 963         if self.use_vendored_sources:
 964             if not os.path.exists('.cargo'):
 965                 os.makedirs('.cargo')
 966             with output('.cargo/config') as cargo_config:
 967                 cargo_config.write(
 968                     "[source.crates-io]\n"
 969                     "replace-with = 'vendored-sources'\n"
 970                     "registry = 'https://example.com'\n"
 971                     "\n"
 972                     "[source.vendored-sources]\n"
 973                     "directory = '{}/vendor'\n"
 974                     .format(self.rust_root))
 975         else:
 976             if os.path.exists('.cargo'):
 977                 shutil.rmtree('.cargo')
 978
 979     def ensure_vendored(self):
 980         """Ensure that the vendored sources are available if needed"""
 981         vendor_dir = os.path.join(self.rust_root, 'vendor')
 982         # Note that this does not handle updating the vendored dependencies if
 983         # the rust git repository is updated. Normal development usually does
 984         # not use vendoring, so hopefully this isn't too much of a problem.
 985         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 986             run([
 987                 self.cargo(),
 988                 "vendor",
 989                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
 990                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
 991             ], verbose=self.verbose, cwd=self.rust_root)
 992
 993
 994 def bootstrap(help_triggered):
 995     """Configure, fetch, build and run the initial bootstrap"""
 996
 997     # If the user is asking for help, let them know that the whole download-and-build
 998     # process has to happen before anything is printed out.
 999     if help_triggered:
1000         print("info: Downloading and building bootstrap before processing --help")
1001         print("      command. See src/bootstrap/README.md for help with common")
1002         print("      commands.")
1003
1004     parser = argparse.ArgumentParser(description='Build rust')
1005     parser.add_argument('--config')
1006     parser.add_argument('--build')
1007     parser.add_argument('--clean', action='store_true')
1008     parser.add_argument('-v', '--verbose', action='count', default=0)
1009
1010     args = [a for a in sys.argv if a != '-h' and a != '--help']
1011     args, _ = parser.parse_known_args(args)
1012
1013     # Configure initial bootstrap
1014     build = RustBuild()
1015     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1016     build.verbose = args.verbose
1017     build.clean = args.clean
1018
1019     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1020     # exists).
1021     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1022     if not toml_path and os.path.exists('config.toml'):
1023         toml_path = 'config.toml'
1024
1025     if toml_path:
1026         if not os.path.exists(toml_path):
1027             toml_path = os.path.join(build.rust_root, toml_path)
1028
1029         with open(toml_path) as config:
1030             build.config_toml = config.read()
1031
1032     profile = build.get_toml('profile')
1033     if profile is not None:
1034         include_file = 'config.{}.toml'.format(profile)
1035         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1036         include_path = os.path.join(include_dir, include_file)
1037         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1038         # specific key, so appending our defaults at the end allows the user to override them
1039         with open(include_path) as included_toml:
1040             build.config_toml += os.linesep + included_toml.read()
1041
1042     config_verbose = build.get_toml('verbose', 'build')
1043     if config_verbose is not None:
1044         build.verbose = max(build.verbose, int(config_verbose))
1045
1046     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1047
1048     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1049
1050     build.check_vendored_status()
1051
1052     build_dir = build.get_toml('build-dir', 'build') or 'build'
1053     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1054
1055     data = stage0_data(build.rust_root)
1056     build.date = data['date']
1057     build.rustc_channel = data['rustc']
1058
1059     if "rustfmt" in data:
1060         build.rustfmt_channel = data['rustfmt']
1061
1062     if 'dev' in data:
1063         build.set_dev_environment()
1064     else:
1065         build.set_normal_environment()
1066
1067     build.update_submodules()
1068
1069     # Fetch/build the bootstrap
1070     build.build = args.build or build.build_triple()
1071     build.download_stage0()
1072     sys.stdout.flush()
1073     build.ensure_vendored()
1074     build.build_bootstrap()
1075     sys.stdout.flush()
1076
1077     # Run the bootstrap
1078     args = [build.bootstrap_binary()]
1079     args.extend(sys.argv[1:])
1080     env = os.environ.copy()
1081     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1082     env["BOOTSTRAP_PYTHON"] = sys.executable
1083     env["BUILD_DIR"] = build.build_dir
1084     env["RUSTC_BOOTSTRAP"] = '1'
1085     if toml_path:
1086         env["BOOTSTRAP_CONFIG"] = toml_path
1087     run(args, env=env, verbose=build.verbose)
1088
1089
1090 def main():
1091     """Entry point for the bootstrap process"""
1092     start_time = time()
1093
1094     # x.py help <cmd> ...
1095     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1096         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1097
1098     help_triggered = (
1099         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1100     try:
1101         bootstrap(help_triggered)
1102         if not help_triggered:
1103             print("Build completed successfully in {}".format(
1104                 format_build_time(time() - start_time)))
1105     except (SystemExit, KeyboardInterrupt) as error:
1106         if hasattr(error, 'code') and isinstance(error.code, int):
1107             exit_code = error.code
1108         else:
1109             exit_code = 1
1110             print(error)
1111         if not help_triggered:
1112             print("Build completed unsuccessfully in {}".format(
1113                 format_build_time(time() - start_time)))
1114         sys.exit(exit_code)
1115
1116
1117 if __name__ == '__main__':
1118     main()