src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple(verbose):
 191     """Build triple as in LLVM"""
 192     # If the user already has a host build triple with an existing `rustc`
 193     # install, use their preference. This fixes most issues with Windows builds
 194     # being detected as GNU instead of MSVC.
 195     default_encoding = sys.getdefaultencoding()
 196     try:
 197         version = subprocess.check_output(["rustc", "--version", "--verbose"])
 198         version = version.decode(default_encoding)
 199         host = next(x for x in version.split('\n') if x.startswith("host: "))
 200         triple = host.split("host: ")[1]
 201         if verbose:
 202             print("detected default triple {}".format(triple))
 203         return triple
 204     except Exception as e:
 205         if verbose:
 206             print("rustup not detected: {}".format(e))
 207             print("falling back to auto-detect")
 208
 209     required = sys.platform != 'win32'
 210     ostype = require(["uname", "-s"], exit=required)
 211     cputype = require(['uname', '-m'], exit=required)
 212
 213     # If we do not have `uname`, assume Windows.
 214     if ostype is None or cputype is None:
 215         return 'x86_64-pc-windows-msvc'
 216
 217     ostype = ostype.decode(default_encoding)
 218     cputype = cputype.decode(default_encoding)
 219
 220     # The goal here is to come up with the same triple as LLVM would,
 221     # at least for the subset of platforms we're willing to target.
 222     ostype_mapper = {
 223         'Darwin': 'apple-darwin',
 224         'DragonFly': 'unknown-dragonfly',
 225         'FreeBSD': 'unknown-freebsd',
 226         'Haiku': 'unknown-haiku',
 227         'NetBSD': 'unknown-netbsd',
 228         'OpenBSD': 'unknown-openbsd'
 229     }
 230
 231     # Consider the direct transformation first and then the special cases
 232     if ostype in ostype_mapper:
 233         ostype = ostype_mapper[ostype]
 234     elif ostype == 'Linux':
 235         os_from_sp = subprocess.check_output(
 236             ['uname', '-o']).strip().decode(default_encoding)
 237         if os_from_sp == 'Android':
 238             ostype = 'linux-android'
 239         else:
 240             ostype = 'unknown-linux-gnu'
 241     elif ostype == 'SunOS':
 242         ostype = 'sun-solaris'
 243         # On Solaris, uname -m will return a machine classification instead
 244         # of a cpu type, so uname -p is recommended instead.  However, the
 245         # output from that option is too generic for our purposes (it will
 246         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 247         # must be used instead.
 248         cputype = require(['isainfo', '-k']).decode(default_encoding)
 249     elif ostype.startswith('MINGW'):
 250         # msys' `uname` does not print gcc configuration, but prints msys
 251         # configuration. so we cannot believe `uname -m`:
 252         # msys1 is always i686 and msys2 is always x86_64.
 253         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 254         # MINGW64 on x86_64.
 255         ostype = 'pc-windows-gnu'
 256         cputype = 'i686'
 257         if os.environ.get('MSYSTEM') == 'MINGW64':
 258             cputype = 'x86_64'
 259     elif ostype.startswith('MSYS'):
 260         ostype = 'pc-windows-gnu'
 261     elif ostype.startswith('CYGWIN_NT'):
 262         cputype = 'i686'
 263         if ostype.endswith('WOW64'):
 264             cputype = 'x86_64'
 265         ostype = 'pc-windows-gnu'
 266     elif sys.platform == 'win32':
 267         # Some Windows platforms might have a `uname` command that returns a
 268         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 269         # these cases, fall back to using sys.platform.
 270         return 'x86_64-pc-windows-msvc'
 271     else:
 272         err = "unknown OS type: {}".format(ostype)
 273         sys.exit(err)
 274
 275     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 276         cputype = subprocess.check_output(
 277               ['uname', '-p']).strip().decode(default_encoding)
 278     cputype_mapper = {
 279         'BePC': 'i686',
 280         'aarch64': 'aarch64',
 281         'amd64': 'x86_64',
 282         'arm64': 'aarch64',
 283         'i386': 'i686',
 284         'i486': 'i686',
 285         'i686': 'i686',
 286         'i786': 'i686',
 287         'powerpc': 'powerpc',
 288         'powerpc64': 'powerpc64',
 289         'powerpc64le': 'powerpc64le',
 290         'ppc': 'powerpc',
 291         'ppc64': 'powerpc64',
 292         'ppc64le': 'powerpc64le',
 293         's390x': 's390x',
 294         'x64': 'x86_64',
 295         'x86': 'i686',
 296         'x86-64': 'x86_64',
 297         'x86_64': 'x86_64'
 298     }
 299
 300     # Consider the direct transformation first and then the special cases
 301     if cputype in cputype_mapper:
 302         cputype = cputype_mapper[cputype]
 303     elif cputype in {'xscale', 'arm'}:
 304         cputype = 'arm'
 305         if ostype == 'linux-android':
 306             ostype = 'linux-androideabi'
 307         elif ostype == 'unknown-freebsd':
 308             cputype = subprocess.check_output(
 309                 ['uname', '-p']).strip().decode(default_encoding)
 310             ostype = 'unknown-freebsd'
 311     elif cputype == 'armv6l':
 312         cputype = 'arm'
 313         if ostype == 'linux-android':
 314             ostype = 'linux-androideabi'
 315         else:
 316             ostype += 'eabihf'
 317     elif cputype in {'armv7l', 'armv8l'}:
 318         cputype = 'armv7'
 319         if ostype == 'linux-android':
 320             ostype = 'linux-androideabi'
 321         else:
 322             ostype += 'eabihf'
 323     elif cputype == 'mips':
 324         if sys.byteorder == 'big':
 325             cputype = 'mips'
 326         elif sys.byteorder == 'little':
 327             cputype = 'mipsel'
 328         else:
 329             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 330     elif cputype == 'mips64':
 331         if sys.byteorder == 'big':
 332             cputype = 'mips64'
 333         elif sys.byteorder == 'little':
 334             cputype = 'mips64el'
 335         else:
 336             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 337         # only the n64 ABI is supported, indicate it
 338         ostype += 'abi64'
 339     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 340         pass
 341     else:
 342         err = "unknown cpu type: {}".format(cputype)
 343         sys.exit(err)
 344
 345     return "{}-{}".format(cputype, ostype)
 346
 347
 348 @contextlib.contextmanager
 349 def output(filepath):
 350     tmp = filepath + '.tmp'
 351     with open(tmp, 'w') as f:
 352         yield f
 353     try:
 354         if os.path.exists(filepath):
 355             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 356     except OSError:
 357         shutil.copy2(tmp, filepath)
 358         os.remove(tmp)
 359         return
 360     os.rename(tmp, filepath)
 361
 362
 363 class RustBuild(object):
 364     """Provide all the methods required to build Rust"""
 365     def __init__(self):
 366         self.date = ''
 367         self._download_url = ''
 368         self.rustc_channel = ''
 369         self.rustfmt_channel = ''
 370         self.build = ''
 371         self.build_dir = ''
 372         self.clean = False
 373         self.config_toml = ''
 374         self.rust_root = ''
 375         self.use_locked_deps = ''
 376         self.use_vendored_sources = ''
 377         self.verbose = False
 378         self.git_version = None
 379         self.nix_deps_dir = None
 380
 381     def download_stage0(self):
 382         """Fetch the build system for Rust, written in Rust
 383
 384         This method will build a cache directory, then it will fetch the
 385         tarball which has the stage0 compiler used to then bootstrap the Rust
 386         compiler itself.
 387
 388         Each downloaded tarball is extracted, after that, the script
 389         will move all the content to the right place.
 390         """
 391         rustc_channel = self.rustc_channel
 392         rustfmt_channel = self.rustfmt_channel
 393
 394         if self.rustc().startswith(self.bin_root()) and \
 395                 (not os.path.exists(self.rustc()) or
 396                  self.program_out_of_date(self.rustc_stamp(), self.date)):
 397             if os.path.exists(self.bin_root()):
 398                 shutil.rmtree(self.bin_root())
 399             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 400             filename = "rust-std-{}-{}{}".format(
 401                 rustc_channel, self.build, tarball_suffix)
 402             pattern = "rust-std-{}".format(self.build)
 403             self._download_stage0_helper(filename, pattern, tarball_suffix)
 404             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 405                                               tarball_suffix)
 406             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 407             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 408                                               tarball_suffix)
 409             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 410             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 411             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 412             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 413             lib_dir = "{}/lib".format(self.bin_root())
 414             for lib in os.listdir(lib_dir):
 415                 if lib.endswith(".so"):
 416                     self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
 417             with output(self.rustc_stamp()) as rust_stamp:
 418                 rust_stamp.write(self.date)
 419
 420         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 421             not os.path.exists(self.rustfmt())
 422             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 423         ):
 424             if rustfmt_channel:
 425                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 426                 [channel, date] = rustfmt_channel.split('-', 1)
 427                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 428                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 429                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 430                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 431                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 432                     rustfmt_stamp.write(self.rustfmt_channel)
 433
 434         if self.downloading_llvm():
 435             # We want the most recent LLVM submodule update to avoid downloading
 436             # LLVM more often than necessary.
 437             #
 438             # This git command finds that commit SHA, looking for bors-authored
 439             # merges that modified src/llvm-project.
 440             #
 441             # This works even in a repository that has not yet initialized
 442             # submodules.
 443             top_level = subprocess.check_output([
 444                 "git", "rev-parse", "--show-toplevel",
 445             ]).decode(sys.getdefaultencoding()).strip()
 446             llvm_sha = subprocess.check_output([
 447                 "git", "log", "--author=bors", "--format=%H", "-n1",
 448                 "-m", "--first-parent",
 449                 "--",
 450                 "{}/src/llvm-project".format(top_level),
 451                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 452             ]).decode(sys.getdefaultencoding()).strip()
 453             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 454             llvm_root = self.llvm_root()
 455             llvm_lib = os.path.join(llvm_root, "lib")
 456             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 457                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 458                 for binary in ["llvm-config", "FileCheck"]:
 459                     self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
 460                 for lib in os.listdir(llvm_lib):
 461                     if lib.endswith(".so"):
 462                         self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
 463                 with output(self.llvm_stamp()) as llvm_stamp:
 464                     llvm_stamp.write(llvm_sha + str(llvm_assertions))
 465
 466     def downloading_llvm(self):
 467         opt = self.get_toml('download-ci-llvm', 'llvm')
 468         return opt == "true" \
 469             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 470
 471     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 472         if date is None:
 473             date = self.date
 474         cache_dst = os.path.join(self.build_dir, "cache")
 475         rustc_cache = os.path.join(cache_dst, date)
 476         if not os.path.exists(rustc_cache):
 477             os.makedirs(rustc_cache)
 478
 479         url = "{}/dist/{}".format(self._download_url, date)
 480         tarball = os.path.join(rustc_cache, filename)
 481         if not os.path.exists(tarball):
 482             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 483         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 484
 485     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 486         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 487         cache_dst = os.path.join(self.build_dir, "cache")
 488         rustc_cache = os.path.join(cache_dst, cache_prefix)
 489         if not os.path.exists(rustc_cache):
 490             os.makedirs(rustc_cache)
 491
 492         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 493         if llvm_assertions:
 494             url = url.replace('rustc-builds', 'rustc-builds-alt')
 495         # ci-artifacts are only stored as .xz, not .gz
 496         if not support_xz():
 497             print("error: XZ support is required to download LLVM")
 498             print("help: consider disabling `download-ci-llvm` or using python3")
 499             exit(1)
 500         tarball_suffix = '.tar.xz'
 501         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 502         tarball = os.path.join(rustc_cache, filename)
 503         if not os.path.exists(tarball):
 504             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 505         unpack(tarball, tarball_suffix, self.llvm_root(),
 506                 match="rust-dev",
 507                 verbose=self.verbose)
 508
 509     def fix_bin_or_dylib(self, fname, rpath_libz=False):
 510         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 511         or the RPATH section, to fix the dynamic library search path
 512
 513         This method is only required on NixOS and uses the PatchELF utility to
 514         change the interpreter/RPATH of ELF executables.
 515
 516         Please see https://nixos.org/patchelf.html for more information
 517         """
 518         default_encoding = sys.getdefaultencoding()
 519         try:
 520             ostype = subprocess.check_output(
 521                 ['uname', '-s']).strip().decode(default_encoding)
 522         except subprocess.CalledProcessError:
 523             return
 524         except OSError as reason:
 525             if getattr(reason, 'winerror', None) is not None:
 526                 return
 527             raise reason
 528
 529         if ostype != "Linux":
 530             return
 531
 532         if not os.path.exists("/etc/NIXOS"):
 533             return
 534         if os.path.exists("/lib"):
 535             return
 536
 537         # At this point we're pretty sure the user is running NixOS
 538         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 539         print(nix_os_msg, fname)
 540
 541         # Only build `stage0/.nix-deps` once.
 542         nix_deps_dir = self.nix_deps_dir
 543         if not nix_deps_dir:
 544             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 545             if not os.path.exists(nix_deps_dir):
 546                 os.makedirs(nix_deps_dir)
 547
 548             nix_deps = [
 549                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 550                 "stdenv.cc.bintools",
 551
 552                 # Needed as a system dependency of `libLLVM-*.so`.
 553                 "zlib",
 554
 555                 # Needed for patching ELF binaries (see doc comment above).
 556                 "patchelf",
 557             ]
 558
 559             # Run `nix-build` to "build" each dependency (which will likely reuse
 560             # the existing `/nix/store` copy, or at most download a pre-built copy).
 561             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 562             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 563             # ensuring garbage collection will never remove the `/nix/store` path
 564             # (which would break our patched binaries that hardcode those paths).
 565             for dep in nix_deps:
 566                 try:
 567                     subprocess.check_output([
 568                         "nix-build", "<nixpkgs>",
 569                         "-A", dep,
 570                         "-o", "{}/{}".format(nix_deps_dir, dep),
 571                     ])
 572                 except subprocess.CalledProcessError as reason:
 573                     print("warning: failed to call nix-build:", reason)
 574                     return
 575
 576             self.nix_deps_dir = nix_deps_dir
 577
 578         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 579         patchelf_args = []
 580
 581         if rpath_libz:
 582             # Patch RPATH to add `zlib` dependency that stems from LLVM
 583             dylib_deps = ["zlib"]
 584             rpath_entries = [
 585                 # Relative default, all binary and dynamic libraries we ship
 586                 # appear to have this (even when `../lib` is redundant).
 587                 "$ORIGIN/../lib",
 588             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 589             patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
 590         if not fname.endswith(".so"):
 591             # Finally, set the corret .interp for binaries
 592             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 593             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 594                 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
 595
 596         try:
 597             subprocess.check_output([patchelf] + patchelf_args + [fname])
 598         except subprocess.CalledProcessError as reason:
 599             print("warning: failed to call patchelf:", reason)
 600             return
 601
 602     def rustc_stamp(self):
 603         """Return the path for .rustc-stamp
 604
 605         >>> rb = RustBuild()
 606         >>> rb.build_dir = "build"
 607         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 608         True
 609         """
 610         return os.path.join(self.bin_root(), '.rustc-stamp')
 611
 612     def rustfmt_stamp(self):
 613         """Return the path for .rustfmt-stamp
 614
 615         >>> rb = RustBuild()
 616         >>> rb.build_dir = "build"
 617         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 618         True
 619         """
 620         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 621
 622     def llvm_stamp(self):
 623         """Return the path for .rustfmt-stamp
 624
 625         >>> rb = RustBuild()
 626         >>> rb.build_dir = "build"
 627         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 628         True
 629         """
 630         return os.path.join(self.llvm_root(), '.llvm-stamp')
 631
 632
 633     def program_out_of_date(self, stamp_path, key):
 634         """Check if the given program stamp is out of date"""
 635         if not os.path.exists(stamp_path) or self.clean:
 636             return True
 637         with open(stamp_path, 'r') as stamp:
 638             return key != stamp.read()
 639
 640     def bin_root(self):
 641         """Return the binary root directory
 642
 643         >>> rb = RustBuild()
 644         >>> rb.build_dir = "build"
 645         >>> rb.bin_root() == os.path.join("build", "stage0")
 646         True
 647
 648         When the 'build' property is given should be a nested directory:
 649
 650         >>> rb.build = "devel"
 651         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 652         True
 653         """
 654         return os.path.join(self.build_dir, self.build, "stage0")
 655
 656     def llvm_root(self):
 657         """Return the CI LLVM root directory
 658
 659         >>> rb = RustBuild()
 660         >>> rb.build_dir = "build"
 661         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 662         True
 663
 664         When the 'build' property is given should be a nested directory:
 665
 666         >>> rb.build = "devel"
 667         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 668         True
 669         """
 670         return os.path.join(self.build_dir, self.build, "ci-llvm")
 671
 672     def get_toml(self, key, section=None):
 673         """Returns the value of the given key in config.toml, otherwise returns None
 674
 675         >>> rb = RustBuild()
 676         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 677         >>> rb.get_toml("key2")
 678         'value2'
 679
 680         If the key does not exists, the result is None:
 681
 682         >>> rb.get_toml("key3") is None
 683         True
 684
 685         Optionally also matches the section the key appears in
 686
 687         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 688         >>> rb.get_toml('key', 'a')
 689         'value1'
 690         >>> rb.get_toml('key', 'b')
 691         'value2'
 692         >>> rb.get_toml('key', 'c') is None
 693         True
 694
 695         >>> rb.config_toml = 'key1 = true'
 696         >>> rb.get_toml("key1")
 697         'true'
 698         """
 699
 700         cur_section = None
 701         for line in self.config_toml.splitlines():
 702             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 703             if section_match is not None:
 704                 cur_section = section_match.group(1)
 705
 706             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 707             if match is not None:
 708                 value = match.group(1)
 709                 if section is None or section == cur_section:
 710                     return self.get_string(value) or value.strip()
 711         return None
 712
 713     def cargo(self):
 714         """Return config path for cargo"""
 715         return self.program_config('cargo')
 716
 717     def rustc(self):
 718         """Return config path for rustc"""
 719         return self.program_config('rustc')
 720
 721     def rustfmt(self):
 722         """Return config path for rustfmt"""
 723         if not self.rustfmt_channel:
 724             return None
 725         return self.program_config('rustfmt')
 726
 727     def program_config(self, program):
 728         """Return config path for the given program
 729
 730         >>> rb = RustBuild()
 731         >>> rb.config_toml = 'rustc = "rustc"\\n'
 732         >>> rb.program_config('rustc')
 733         'rustc'
 734         >>> rb.config_toml = ''
 735         >>> cargo_path = rb.program_config('cargo')
 736         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 737         ... "bin", "cargo")
 738         True
 739         """
 740         config = self.get_toml(program)
 741         if config:
 742             return os.path.expanduser(config)
 743         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 744             program, self.exe_suffix()))
 745
 746     @staticmethod
 747     def get_string(line):
 748         """Return the value between double quotes
 749
 750         >>> RustBuild.get_string('    "devel"   ')
 751         'devel'
 752         >>> RustBuild.get_string("    'devel'   ")
 753         'devel'
 754         >>> RustBuild.get_string('devel') is None
 755         True
 756         >>> RustBuild.get_string('    "devel   ')
 757         ''
 758         """
 759         start = line.find('"')
 760         if start != -1:
 761             end = start + 1 + line[start + 1:].find('"')
 762             return line[start + 1:end]
 763         start = line.find('\'')
 764         if start != -1:
 765             end = start + 1 + line[start + 1:].find('\'')
 766             return line[start + 1:end]
 767         return None
 768
 769     @staticmethod
 770     def exe_suffix():
 771         """Return a suffix for executables"""
 772         if sys.platform == 'win32':
 773             return '.exe'
 774         return ''
 775
 776     def bootstrap_binary(self):
 777         """Return the path of the bootstrap binary
 778
 779         >>> rb = RustBuild()
 780         >>> rb.build_dir = "build"
 781         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 782         ... "debug", "bootstrap")
 783         True
 784         """
 785         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 786
 787     def build_bootstrap(self):
 788         """Build bootstrap"""
 789         build_dir = os.path.join(self.build_dir, "bootstrap")
 790         if self.clean and os.path.exists(build_dir):
 791             shutil.rmtree(build_dir)
 792         env = os.environ.copy()
 793         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 794         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 795         if "CARGO_BUILD_TARGET" in env:
 796             del env["CARGO_BUILD_TARGET"]
 797         env["CARGO_TARGET_DIR"] = build_dir
 798         env["RUSTC"] = self.rustc()
 799         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 800             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 801             if "LD_LIBRARY_PATH" in env else ""
 802         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 803             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 804             if "DYLD_LIBRARY_PATH" in env else ""
 805         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 806             (os.pathsep + env["LIBRARY_PATH"]) \
 807             if "LIBRARY_PATH" in env else ""
 808         # preserve existing RUSTFLAGS
 809         env.setdefault("RUSTFLAGS", "")
 810         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 811
 812         build_section = "target.{}".format(self.build)
 813         target_features = []
 814         if self.get_toml("crt-static", build_section) == "true":
 815             target_features += ["+crt-static"]
 816         elif self.get_toml("crt-static", build_section) == "false":
 817             target_features += ["-crt-static"]
 818         if target_features:
 819             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 820         target_linker = self.get_toml("linker", build_section)
 821         if target_linker is not None:
 822             env["RUSTFLAGS"] += " -C linker=" + target_linker
 823         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 824         if self.get_toml("deny-warnings", "rust") != "false":
 825             env["RUSTFLAGS"] += " -Dwarnings"
 826
 827         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 828             os.pathsep + env["PATH"]
 829         if not os.path.isfile(self.cargo()):
 830             raise Exception("no cargo executable found at `{}`".format(
 831                 self.cargo()))
 832         args = [self.cargo(), "build", "--manifest-path",
 833                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 834         for _ in range(1, self.verbose):
 835             args.append("--verbose")
 836         if self.use_locked_deps:
 837             args.append("--locked")
 838         if self.use_vendored_sources:
 839             args.append("--frozen")
 840         run(args, env=env, verbose=self.verbose)
 841
 842     def build_triple(self):
 843         """Build triple as in LLVM
 844
 845         Note that `default_build_triple` is moderately expensive,
 846         so use `self.build` where possible.
 847         """
 848         config = self.get_toml('build')
 849         if config:
 850             return config
 851         return default_build_triple(self.verbose)
 852
 853     def check_submodule(self, module, slow_submodules):
 854         if not slow_submodules:
 855             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 856                                            cwd=os.path.join(self.rust_root, module),
 857                                            stdout=subprocess.PIPE)
 858             return checked_out
 859         else:
 860             return None
 861
 862     def update_submodule(self, module, checked_out, recorded_submodules):
 863         module_path = os.path.join(self.rust_root, module)
 864
 865         if checked_out is not None:
 866             default_encoding = sys.getdefaultencoding()
 867             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 868             if recorded_submodules[module] == checked_out:
 869                 return
 870
 871         print("Updating submodule", module)
 872
 873         run(["git", "submodule", "-q", "sync", module],
 874             cwd=self.rust_root, verbose=self.verbose)
 875
 876         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 877         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 878             update_args.append("--progress")
 879         update_args.append(module)
 880         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 881
 882         run(["git", "reset", "-q", "--hard"],
 883             cwd=module_path, verbose=self.verbose)
 884         run(["git", "clean", "-qdfx"],
 885             cwd=module_path, verbose=self.verbose)
 886
 887     def update_submodules(self):
 888         """Update submodules"""
 889         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 890                 self.get_toml('submodules') == "false":
 891             return
 892
 893         default_encoding = sys.getdefaultencoding()
 894
 895         # check the existence and version of 'git' command
 896         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 897         self.git_version = distutils.version.LooseVersion(git_version_str)
 898
 899         slow_submodules = self.get_toml('fast-submodules') == "false"
 900         start_time = time()
 901         if slow_submodules:
 902             print('Unconditionally updating all submodules')
 903         else:
 904             print('Updating only changed submodules')
 905         default_encoding = sys.getdefaultencoding()
 906         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 907             ["git", "config", "--file",
 908              os.path.join(self.rust_root, ".gitmodules"),
 909              "--get-regexp", "path"]
 910         ).decode(default_encoding).splitlines()]
 911         filtered_submodules = []
 912         submodules_names = []
 913         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
 914         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
 915         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
 916             or "llvm" in self.get_toml('codegen-backends', 'rust')
 917         for module in submodules:
 918             if module.endswith("llvm-project"):
 919                 # Don't sync the llvm-project submodule if an external LLVM was
 920                 # provided, if we are downloading LLVM or if the LLVM backend is
 921                 # not being built. Also, if the submodule has been initialized
 922                 # already, sync it anyways so that it doesn't mess up contributor
 923                 # pull requests.
 924                 if external_llvm_provided or not llvm_needed:
 925                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
 926                         continue
 927             check = self.check_submodule(module, slow_submodules)
 928             filtered_submodules.append((module, check))
 929             submodules_names.append(module)
 930         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 931                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 932         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 933         recorded_submodules = {}
 934         for data in recorded:
 935             data = data.split()
 936             recorded_submodules[data[3]] = data[2]
 937         for module in filtered_submodules:
 938             self.update_submodule(module[0], module[1], recorded_submodules)
 939         print("Submodules updated in %.2f seconds" % (time() - start_time))
 940
 941     def set_normal_environment(self):
 942         """Set download URL for normal environment"""
 943         if 'RUSTUP_DIST_SERVER' in os.environ:
 944             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 945         else:
 946             self._download_url = 'https://static.rust-lang.org'
 947
 948     def set_dev_environment(self):
 949         """Set download URL for development environment"""
 950         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 951             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 952         else:
 953             self._download_url = 'https://dev-static.rust-lang.org'
 954
 955     def check_vendored_status(self):
 956         """Check that vendoring is configured properly"""
 957         vendor_dir = os.path.join(self.rust_root, 'vendor')
 958         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 959             if os.environ.get('USER') != os.environ['SUDO_USER']:
 960                 self.use_vendored_sources = True
 961                 print('info: looks like you are running this command under `sudo`')
 962                 print('      and so in order to preserve your $HOME this will now')
 963                 print('      use vendored sources by default.')
 964                 if not os.path.exists(vendor_dir):
 965                     print('error: vendoring required, but vendor directory does not exist.')
 966                     print('       Run `cargo vendor` without sudo to initialize the '
 967                           'vendor directory.')
 968                     raise Exception("{} not found".format(vendor_dir))
 969
 970         if self.use_vendored_sources:
 971             if not os.path.exists('.cargo'):
 972                 os.makedirs('.cargo')
 973             with output('.cargo/config') as cargo_config:
 974                 cargo_config.write(
 975                     "[source.crates-io]\n"
 976                     "replace-with = 'vendored-sources'\n"
 977                     "registry = 'https://example.com'\n"
 978                     "\n"
 979                     "[source.vendored-sources]\n"
 980                     "directory = '{}/vendor'\n"
 981                     .format(self.rust_root))
 982         else:
 983             if os.path.exists('.cargo'):
 984                 shutil.rmtree('.cargo')
 985
 986     def ensure_vendored(self):
 987         """Ensure that the vendored sources are available if needed"""
 988         vendor_dir = os.path.join(self.rust_root, 'vendor')
 989         # Note that this does not handle updating the vendored dependencies if
 990         # the rust git repository is updated. Normal development usually does
 991         # not use vendoring, so hopefully this isn't too much of a problem.
 992         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 993             run([
 994                 self.cargo(),
 995                 "vendor",
 996                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
 997                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
 998             ], verbose=self.verbose, cwd=self.rust_root)
 999
1000
1001 def bootstrap(help_triggered):
1002     """Configure, fetch, build and run the initial bootstrap"""
1003
1004     # If the user is asking for help, let them know that the whole download-and-build
1005     # process has to happen before anything is printed out.
1006     if help_triggered:
1007         print("info: Downloading and building bootstrap before processing --help")
1008         print("      command. See src/bootstrap/README.md for help with common")
1009         print("      commands.")
1010
1011     parser = argparse.ArgumentParser(description='Build rust')
1012     parser.add_argument('--config')
1013     parser.add_argument('--build')
1014     parser.add_argument('--clean', action='store_true')
1015     parser.add_argument('-v', '--verbose', action='count', default=0)
1016
1017     args = [a for a in sys.argv if a != '-h' and a != '--help']
1018     args, _ = parser.parse_known_args(args)
1019
1020     # Configure initial bootstrap
1021     build = RustBuild()
1022     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1023     build.verbose = args.verbose
1024     build.clean = args.clean
1025
1026     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1027     # exists).
1028     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1029     if not toml_path and os.path.exists('config.toml'):
1030         toml_path = 'config.toml'
1031
1032     if toml_path:
1033         if not os.path.exists(toml_path):
1034             toml_path = os.path.join(build.rust_root, toml_path)
1035
1036         with open(toml_path) as config:
1037             build.config_toml = config.read()
1038
1039     profile = build.get_toml('profile')
1040     if profile is not None:
1041         include_file = 'config.{}.toml'.format(profile)
1042         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1043         include_path = os.path.join(include_dir, include_file)
1044         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1045         # specific key, so appending our defaults at the end allows the user to override them
1046         with open(include_path) as included_toml:
1047             build.config_toml += os.linesep + included_toml.read()
1048
1049     config_verbose = build.get_toml('verbose', 'build')
1050     if config_verbose is not None:
1051         build.verbose = max(build.verbose, int(config_verbose))
1052
1053     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1054
1055     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1056
1057     build.check_vendored_status()
1058
1059     build_dir = build.get_toml('build-dir', 'build') or 'build'
1060     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1061
1062     data = stage0_data(build.rust_root)
1063     build.date = data['date']
1064     build.rustc_channel = data['rustc']
1065
1066     if "rustfmt" in data:
1067         build.rustfmt_channel = data['rustfmt']
1068
1069     if 'dev' in data:
1070         build.set_dev_environment()
1071     else:
1072         build.set_normal_environment()
1073
1074     build.update_submodules()
1075
1076     # Fetch/build the bootstrap
1077     build.build = args.build or build.build_triple()
1078     build.download_stage0()
1079     sys.stdout.flush()
1080     build.ensure_vendored()
1081     build.build_bootstrap()
1082     sys.stdout.flush()
1083
1084     # Run the bootstrap
1085     args = [build.bootstrap_binary()]
1086     args.extend(sys.argv[1:])
1087     env = os.environ.copy()
1088     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1089     env["BOOTSTRAP_PYTHON"] = sys.executable
1090     env["BUILD_DIR"] = build.build_dir
1091     env["RUSTC_BOOTSTRAP"] = '1'
1092     if toml_path:
1093         env["BOOTSTRAP_CONFIG"] = toml_path
1094     run(args, env=env, verbose=build.verbose)
1095
1096
1097 def main():
1098     """Entry point for the bootstrap process"""
1099     start_time = time()
1100
1101     # x.py help <cmd> ...
1102     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1103         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1104
1105     help_triggered = (
1106         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1107     try:
1108         bootstrap(help_triggered)
1109         if not help_triggered:
1110             print("Build completed successfully in {}".format(
1111                 format_build_time(time() - start_time)))
1112     except (SystemExit, KeyboardInterrupt) as error:
1113         if hasattr(error, 'code') and isinstance(error.code, int):
1114             exit_code = error.code
1115         else:
1116             exit_code = 1
1117             print(error)
1118         if not help_triggered:
1119             print("Build completed unsuccessfully in {}".format(
1120                 format_build_time(time() - start_time)))
1121         sys.exit(exit_code)
1122
1123
1124 if __name__ == '__main__':
1125     main()