src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple():
 191     """Build triple as in LLVM"""
 192     default_encoding = sys.getdefaultencoding()
 193     required = sys.platform != 'win32'
 194     ostype = require(["uname", "-s"], exit=required)
 195     cputype = require(['uname', '-m'], exit=required)
 196
 197     # If we do not have `uname`, assume Windows.
 198     if ostype is None or cputype is None:
 199         return 'x86_64-pc-windows-msvc'
 200
 201     ostype = ostype.decode(default_encoding)
 202     cputype = cputype.decode(default_encoding)
 203
 204     # The goal here is to come up with the same triple as LLVM would,
 205     # at least for the subset of platforms we're willing to target.
 206     ostype_mapper = {
 207         'Darwin': 'apple-darwin',
 208         'DragonFly': 'unknown-dragonfly',
 209         'FreeBSD': 'unknown-freebsd',
 210         'Haiku': 'unknown-haiku',
 211         'NetBSD': 'unknown-netbsd',
 212         'OpenBSD': 'unknown-openbsd'
 213     }
 214
 215     # Consider the direct transformation first and then the special cases
 216     if ostype in ostype_mapper:
 217         ostype = ostype_mapper[ostype]
 218     elif ostype == 'Linux':
 219         os_from_sp = subprocess.check_output(
 220             ['uname', '-o']).strip().decode(default_encoding)
 221         if os_from_sp == 'Android':
 222             ostype = 'linux-android'
 223         else:
 224             ostype = 'unknown-linux-gnu'
 225     elif ostype == 'SunOS':
 226         ostype = 'sun-solaris'
 227         # On Solaris, uname -m will return a machine classification instead
 228         # of a cpu type, so uname -p is recommended instead.  However, the
 229         # output from that option is too generic for our purposes (it will
 230         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 231         # must be used instead.
 232         cputype = require(['isainfo', '-k']).decode(default_encoding)
 233     elif ostype.startswith('MINGW'):
 234         # msys' `uname` does not print gcc configuration, but prints msys
 235         # configuration. so we cannot believe `uname -m`:
 236         # msys1 is always i686 and msys2 is always x86_64.
 237         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 238         # MINGW64 on x86_64.
 239         ostype = 'pc-windows-gnu'
 240         cputype = 'i686'
 241         if os.environ.get('MSYSTEM') == 'MINGW64':
 242             cputype = 'x86_64'
 243     elif ostype.startswith('MSYS'):
 244         ostype = 'pc-windows-gnu'
 245     elif ostype.startswith('CYGWIN_NT'):
 246         cputype = 'i686'
 247         if ostype.endswith('WOW64'):
 248             cputype = 'x86_64'
 249         ostype = 'pc-windows-gnu'
 250     elif sys.platform == 'win32':
 251         # Some Windows platforms might have a `uname` command that returns a
 252         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 253         # these cases, fall back to using sys.platform.
 254         return 'x86_64-pc-windows-msvc'
 255     else:
 256         err = "unknown OS type: {}".format(ostype)
 257         sys.exit(err)
 258
 259     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 260         cputype = subprocess.check_output(
 261               ['uname', '-p']).strip().decode(default_encoding)
 262     cputype_mapper = {
 263         'BePC': 'i686',
 264         'aarch64': 'aarch64',
 265         'amd64': 'x86_64',
 266         'arm64': 'aarch64',
 267         'i386': 'i686',
 268         'i486': 'i686',
 269         'i686': 'i686',
 270         'i786': 'i686',
 271         'powerpc': 'powerpc',
 272         'powerpc64': 'powerpc64',
 273         'powerpc64le': 'powerpc64le',
 274         'ppc': 'powerpc',
 275         'ppc64': 'powerpc64',
 276         'ppc64le': 'powerpc64le',
 277         's390x': 's390x',
 278         'x64': 'x86_64',
 279         'x86': 'i686',
 280         'x86-64': 'x86_64',
 281         'x86_64': 'x86_64'
 282     }
 283
 284     # Consider the direct transformation first and then the special cases
 285     if cputype in cputype_mapper:
 286         cputype = cputype_mapper[cputype]
 287     elif cputype in {'xscale', 'arm'}:
 288         cputype = 'arm'
 289         if ostype == 'linux-android':
 290             ostype = 'linux-androideabi'
 291         elif ostype == 'unknown-freebsd':
 292             cputype = subprocess.check_output(
 293                 ['uname', '-p']).strip().decode(default_encoding)
 294             ostype = 'unknown-freebsd'
 295     elif cputype == 'armv6l':
 296         cputype = 'arm'
 297         if ostype == 'linux-android':
 298             ostype = 'linux-androideabi'
 299         else:
 300             ostype += 'eabihf'
 301     elif cputype in {'armv7l', 'armv8l'}:
 302         cputype = 'armv7'
 303         if ostype == 'linux-android':
 304             ostype = 'linux-androideabi'
 305         else:
 306             ostype += 'eabihf'
 307     elif cputype == 'mips':
 308         if sys.byteorder == 'big':
 309             cputype = 'mips'
 310         elif sys.byteorder == 'little':
 311             cputype = 'mipsel'
 312         else:
 313             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 314     elif cputype == 'mips64':
 315         if sys.byteorder == 'big':
 316             cputype = 'mips64'
 317         elif sys.byteorder == 'little':
 318             cputype = 'mips64el'
 319         else:
 320             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 321         # only the n64 ABI is supported, indicate it
 322         ostype += 'abi64'
 323     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 324         pass
 325     else:
 326         err = "unknown cpu type: {}".format(cputype)
 327         sys.exit(err)
 328
 329     return "{}-{}".format(cputype, ostype)
 330
 331
 332 @contextlib.contextmanager
 333 def output(filepath):
 334     tmp = filepath + '.tmp'
 335     with open(tmp, 'w') as f:
 336         yield f
 337     try:
 338         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 339         os.rename(tmp, filepath)
 340     except OSError:
 341         shutil.copy2(tmp, filepath)
 342         os.remove(tmp)
 343
 344
 345 class RustBuild(object):
 346     """Provide all the methods required to build Rust"""
 347     def __init__(self):
 348         self.cargo_channel = ''
 349         self.date = ''
 350         self._download_url = ''
 351         self.rustc_channel = ''
 352         self.rustfmt_channel = ''
 353         self.build = ''
 354         self.build_dir = ''
 355         self.clean = False
 356         self.config_toml = ''
 357         self.rust_root = ''
 358         self.use_locked_deps = ''
 359         self.use_vendored_sources = ''
 360         self.verbose = False
 361         self.git_version = None
 362         self.nix_deps_dir = None
 363
 364     def download_stage0(self):
 365         """Fetch the build system for Rust, written in Rust
 366
 367         This method will build a cache directory, then it will fetch the
 368         tarball which has the stage0 compiler used to then bootstrap the Rust
 369         compiler itself.
 370
 371         Each downloaded tarball is extracted, after that, the script
 372         will move all the content to the right place.
 373         """
 374         rustc_channel = self.rustc_channel
 375         cargo_channel = self.cargo_channel
 376         rustfmt_channel = self.rustfmt_channel
 377
 378         if self.rustc().startswith(self.bin_root()) and \
 379                 (not os.path.exists(self.rustc()) or
 380                  self.program_out_of_date(self.rustc_stamp())):
 381             if os.path.exists(self.bin_root()):
 382                 shutil.rmtree(self.bin_root())
 383             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 384             filename = "rust-std-{}-{}{}".format(
 385                 rustc_channel, self.build, tarball_suffix)
 386             pattern = "rust-std-{}".format(self.build)
 387             self._download_stage0_helper(filename, pattern, tarball_suffix)
 388
 389             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 392             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 393             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 394             lib_dir = "{}/lib".format(self.bin_root())
 395             for lib in os.listdir(lib_dir):
 396                 if lib.endswith(".so"):
 397                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 398             with output(self.rustc_stamp()) as rust_stamp:
 399                 rust_stamp.write(self.date)
 400
 401         if self.cargo().startswith(self.bin_root()) and \
 402                 (not os.path.exists(self.cargo()) or
 403                  self.program_out_of_date(self.cargo_stamp())):
 404             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 405             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 406                                               tarball_suffix)
 407             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 408             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 409             with output(self.cargo_stamp()) as cargo_stamp:
 410                 cargo_stamp.write(self.date)
 411
 412         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 413             not os.path.exists(self.rustfmt())
 414             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 415         ):
 416             if rustfmt_channel:
 417                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 418                 [channel, date] = rustfmt_channel.split('-', 1)
 419                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 420                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 421                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 422                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 423                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 424                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 425
 426         if self.downloading_llvm():
 427             # We want the most recent LLVM submodule update to avoid downloading
 428             # LLVM more often than necessary.
 429             #
 430             # This git command finds that commit SHA, looking for bors-authored
 431             # merges that modified src/llvm-project.
 432             #
 433             # This works even in a repository that has not yet initialized
 434             # submodules.
 435             llvm_sha = subprocess.check_output([
 436                 "git", "log", "--author=bors", "--format=%H", "-n1",
 437                 "-m", "--first-parent",
 438                 "--", "src/llvm-project"
 439             ]).decode(sys.getdefaultencoding()).strip()
 440             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 441             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 442                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 443                 for binary in ["llvm-config", "FileCheck"]:
 444                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 445                 with output(self.llvm_stamp()) as llvm_stamp:
 446                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 447
 448     def downloading_llvm(self):
 449         opt = self.get_toml('download-ci-llvm', 'llvm')
 450         return opt == "true" \
 451             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 452
 453     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 454         if date is None:
 455             date = self.date
 456         cache_dst = os.path.join(self.build_dir, "cache")
 457         rustc_cache = os.path.join(cache_dst, date)
 458         if not os.path.exists(rustc_cache):
 459             os.makedirs(rustc_cache)
 460
 461         url = "{}/dist/{}".format(self._download_url, date)
 462         tarball = os.path.join(rustc_cache, filename)
 463         if not os.path.exists(tarball):
 464             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 465         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 466
 467     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 468         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 469         cache_dst = os.path.join(self.build_dir, "cache")
 470         rustc_cache = os.path.join(cache_dst, cache_prefix)
 471         if not os.path.exists(rustc_cache):
 472             os.makedirs(rustc_cache)
 473
 474         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 475         if llvm_assertions:
 476             url = url.replace('rustc-builds', 'rustc-builds-alt')
 477         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 478         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 479         tarball = os.path.join(rustc_cache, filename)
 480         if not os.path.exists(tarball):
 481             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 482         unpack(tarball, tarball_suffix, self.llvm_root(),
 483                 match="rust-dev",
 484                 verbose=self.verbose)
 485
 486     def fix_bin_or_dylib(self, fname):
 487         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 488         or the RPATH section, to fix the dynamic library search path
 489
 490         This method is only required on NixOS and uses the PatchELF utility to
 491         change the interpreter/RPATH of ELF executables.
 492
 493         Please see https://nixos.org/patchelf.html for more information
 494         """
 495         default_encoding = sys.getdefaultencoding()
 496         try:
 497             ostype = subprocess.check_output(
 498                 ['uname', '-s']).strip().decode(default_encoding)
 499         except subprocess.CalledProcessError:
 500             return
 501         except OSError as reason:
 502             if getattr(reason, 'winerror', None) is not None:
 503                 return
 504             raise reason
 505
 506         if ostype != "Linux":
 507             return
 508
 509         if not os.path.exists("/etc/NIXOS"):
 510             return
 511         if os.path.exists("/lib"):
 512             return
 513
 514         # At this point we're pretty sure the user is running NixOS
 515         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 516         print(nix_os_msg, fname)
 517
 518         # Only build `stage0/.nix-deps` once.
 519         nix_deps_dir = self.nix_deps_dir
 520         if not nix_deps_dir:
 521             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 522             if not os.path.exists(nix_deps_dir):
 523                 os.makedirs(nix_deps_dir)
 524
 525             nix_deps = [
 526                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 527                 "stdenv.cc.bintools",
 528
 529                 # Needed as a system dependency of `libLLVM-*.so`.
 530                 "zlib",
 531
 532                 # Needed for patching ELF binaries (see doc comment above).
 533                 "patchelf",
 534             ]
 535
 536             # Run `nix-build` to "build" each dependency (which will likely reuse
 537             # the existing `/nix/store` copy, or at most download a pre-built copy).
 538             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 539             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 540             # ensuring garbage collection will never remove the `/nix/store` path
 541             # (which would break our patched binaries that hardcode those paths).
 542             for dep in nix_deps:
 543                 try:
 544                     subprocess.check_output([
 545                         "nix-build", "<nixpkgs>",
 546                         "-A", dep,
 547                         "-o", "{}/{}".format(nix_deps_dir, dep),
 548                     ])
 549                 except subprocess.CalledProcessError as reason:
 550                     print("warning: failed to call nix-build:", reason)
 551                     return
 552
 553             self.nix_deps_dir = nix_deps_dir
 554
 555         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 556
 557         if fname.endswith(".so"):
 558             # Dynamic library, patch RPATH to point to system dependencies.
 559             dylib_deps = ["zlib"]
 560             rpath_entries = [
 561                 # Relative default, all binary and dynamic libraries we ship
 562                 # appear to have this (even when `../lib` is redundant).
 563                 "$ORIGIN/../lib",
 564             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 565             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 566         else:
 567             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 568             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 569                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 570
 571         try:
 572             subprocess.check_output([patchelf] + patchelf_args + [fname])
 573         except subprocess.CalledProcessError as reason:
 574             print("warning: failed to call patchelf:", reason)
 575             return
 576
 577     def rustc_stamp(self):
 578         """Return the path for .rustc-stamp
 579
 580         >>> rb = RustBuild()
 581         >>> rb.build_dir = "build"
 582         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 583         True
 584         """
 585         return os.path.join(self.bin_root(), '.rustc-stamp')
 586
 587     def cargo_stamp(self):
 588         """Return the path for .cargo-stamp
 589
 590         >>> rb = RustBuild()
 591         >>> rb.build_dir = "build"
 592         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 593         True
 594         """
 595         return os.path.join(self.bin_root(), '.cargo-stamp')
 596
 597     def rustfmt_stamp(self):
 598         """Return the path for .rustfmt-stamp
 599
 600         >>> rb = RustBuild()
 601         >>> rb.build_dir = "build"
 602         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 603         True
 604         """
 605         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 606
 607     def llvm_stamp(self):
 608         """Return the path for .rustfmt-stamp
 609
 610         >>> rb = RustBuild()
 611         >>> rb.build_dir = "build"
 612         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 613         True
 614         """
 615         return os.path.join(self.llvm_root(), '.llvm-stamp')
 616
 617
 618     def program_out_of_date(self, stamp_path, extra=""):
 619         """Check if the given program stamp is out of date"""
 620         if not os.path.exists(stamp_path) or self.clean:
 621             return True
 622         with open(stamp_path, 'r') as stamp:
 623             return (self.date + extra) != stamp.read()
 624
 625     def bin_root(self):
 626         """Return the binary root directory
 627
 628         >>> rb = RustBuild()
 629         >>> rb.build_dir = "build"
 630         >>> rb.bin_root() == os.path.join("build", "stage0")
 631         True
 632
 633         When the 'build' property is given should be a nested directory:
 634
 635         >>> rb.build = "devel"
 636         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 637         True
 638         """
 639         return os.path.join(self.build_dir, self.build, "stage0")
 640
 641     def llvm_root(self):
 642         """Return the CI LLVM root directory
 643
 644         >>> rb = RustBuild()
 645         >>> rb.build_dir = "build"
 646         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 647         True
 648
 649         When the 'build' property is given should be a nested directory:
 650
 651         >>> rb.build = "devel"
 652         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 653         True
 654         """
 655         return os.path.join(self.build_dir, self.build, "ci-llvm")
 656
 657     def get_toml(self, key, section=None):
 658         """Returns the value of the given key in config.toml, otherwise returns None
 659
 660         >>> rb = RustBuild()
 661         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 662         >>> rb.get_toml("key2")
 663         'value2'
 664
 665         If the key does not exists, the result is None:
 666
 667         >>> rb.get_toml("key3") is None
 668         True
 669
 670         Optionally also matches the section the key appears in
 671
 672         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 673         >>> rb.get_toml('key', 'a')
 674         'value1'
 675         >>> rb.get_toml('key', 'b')
 676         'value2'
 677         >>> rb.get_toml('key', 'c') is None
 678         True
 679
 680         >>> rb.config_toml = 'key1 = true'
 681         >>> rb.get_toml("key1")
 682         'true'
 683         """
 684
 685         cur_section = None
 686         for line in self.config_toml.splitlines():
 687             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 688             if section_match is not None:
 689                 cur_section = section_match.group(1)
 690
 691             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 692             if match is not None:
 693                 value = match.group(1)
 694                 if section is None or section == cur_section:
 695                     return self.get_string(value) or value.strip()
 696         return None
 697
 698     def cargo(self):
 699         """Return config path for cargo"""
 700         return self.program_config('cargo')
 701
 702     def rustc(self):
 703         """Return config path for rustc"""
 704         return self.program_config('rustc')
 705
 706     def rustfmt(self):
 707         """Return config path for rustfmt"""
 708         if not self.rustfmt_channel:
 709             return None
 710         return self.program_config('rustfmt')
 711
 712     def program_config(self, program):
 713         """Return config path for the given program
 714
 715         >>> rb = RustBuild()
 716         >>> rb.config_toml = 'rustc = "rustc"\\n'
 717         >>> rb.program_config('rustc')
 718         'rustc'
 719         >>> rb.config_toml = ''
 720         >>> cargo_path = rb.program_config('cargo')
 721         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 722         ... "bin", "cargo")
 723         True
 724         """
 725         config = self.get_toml(program)
 726         if config:
 727             return os.path.expanduser(config)
 728         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 729             program, self.exe_suffix()))
 730
 731     @staticmethod
 732     def get_string(line):
 733         """Return the value between double quotes
 734
 735         >>> RustBuild.get_string('    "devel"   ')
 736         'devel'
 737         >>> RustBuild.get_string("    'devel'   ")
 738         'devel'
 739         >>> RustBuild.get_string('devel') is None
 740         True
 741         >>> RustBuild.get_string('    "devel   ')
 742         ''
 743         """
 744         start = line.find('"')
 745         if start != -1:
 746             end = start + 1 + line[start + 1:].find('"')
 747             return line[start + 1:end]
 748         start = line.find('\'')
 749         if start != -1:
 750             end = start + 1 + line[start + 1:].find('\'')
 751             return line[start + 1:end]
 752         return None
 753
 754     @staticmethod
 755     def exe_suffix():
 756         """Return a suffix for executables"""
 757         if sys.platform == 'win32':
 758             return '.exe'
 759         return ''
 760
 761     def bootstrap_binary(self):
 762         """Return the path of the bootstrap binary
 763
 764         >>> rb = RustBuild()
 765         >>> rb.build_dir = "build"
 766         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 767         ... "debug", "bootstrap")
 768         True
 769         """
 770         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 771
 772     def build_bootstrap(self):
 773         """Build bootstrap"""
 774         build_dir = os.path.join(self.build_dir, "bootstrap")
 775         if self.clean and os.path.exists(build_dir):
 776             shutil.rmtree(build_dir)
 777         env = os.environ.copy()
 778         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 779         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 780         if "CARGO_BUILD_TARGET" in env:
 781             del env["CARGO_BUILD_TARGET"]
 782         env["CARGO_TARGET_DIR"] = build_dir
 783         env["RUSTC"] = self.rustc()
 784         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 785             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 786             if "LD_LIBRARY_PATH" in env else ""
 787         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 788             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 789             if "DYLD_LIBRARY_PATH" in env else ""
 790         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 791             (os.pathsep + env["LIBRARY_PATH"]) \
 792             if "LIBRARY_PATH" in env else ""
 793         # preserve existing RUSTFLAGS
 794         env.setdefault("RUSTFLAGS", "")
 795         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 796
 797         build_section = "target.{}".format(self.build_triple())
 798         target_features = []
 799         if self.get_toml("crt-static", build_section) == "true":
 800             target_features += ["+crt-static"]
 801         elif self.get_toml("crt-static", build_section) == "false":
 802             target_features += ["-crt-static"]
 803         if target_features:
 804             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 805         target_linker = self.get_toml("linker", build_section)
 806         if target_linker is not None:
 807             env["RUSTFLAGS"] += " -C linker=" + target_linker
 808         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 809         if self.get_toml("deny-warnings", "rust") != "false":
 810             env["RUSTFLAGS"] += " -Dwarnings"
 811
 812         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 813             os.pathsep + env["PATH"]
 814         if not os.path.isfile(self.cargo()):
 815             raise Exception("no cargo executable found at `{}`".format(
 816                 self.cargo()))
 817         args = [self.cargo(), "build", "--manifest-path",
 818                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 819         for _ in range(1, self.verbose):
 820             args.append("--verbose")
 821         if self.use_locked_deps:
 822             args.append("--locked")
 823         if self.use_vendored_sources:
 824             args.append("--frozen")
 825         run(args, env=env, verbose=self.verbose)
 826
 827     def build_triple(self):
 828         """Build triple as in LLVM"""
 829         config = self.get_toml('build')
 830         if config:
 831             return config
 832         return default_build_triple()
 833
 834     def check_submodule(self, module, slow_submodules):
 835         if not slow_submodules:
 836             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 837                                            cwd=os.path.join(self.rust_root, module),
 838                                            stdout=subprocess.PIPE)
 839             return checked_out
 840         else:
 841             return None
 842
 843     def update_submodule(self, module, checked_out, recorded_submodules):
 844         module_path = os.path.join(self.rust_root, module)
 845
 846         if checked_out is not None:
 847             default_encoding = sys.getdefaultencoding()
 848             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 849             if recorded_submodules[module] == checked_out:
 850                 return
 851
 852         print("Updating submodule", module)
 853
 854         run(["git", "submodule", "-q", "sync", module],
 855             cwd=self.rust_root, verbose=self.verbose)
 856
 857         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 858         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 859             update_args.append("--progress")
 860         update_args.append(module)
 861         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 862
 863         run(["git", "reset", "-q", "--hard"],
 864             cwd=module_path, verbose=self.verbose)
 865         run(["git", "clean", "-qdfx"],
 866             cwd=module_path, verbose=self.verbose)
 867
 868     def update_submodules(self):
 869         """Update submodules"""
 870         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 871                 self.get_toml('submodules') == "false":
 872             return
 873
 874         default_encoding = sys.getdefaultencoding()
 875
 876         # check the existence and version of 'git' command
 877         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 878         self.git_version = distutils.version.LooseVersion(git_version_str)
 879
 880         slow_submodules = self.get_toml('fast-submodules') == "false"
 881         start_time = time()
 882         if slow_submodules:
 883             print('Unconditionally updating all submodules')
 884         else:
 885             print('Updating only changed submodules')
 886         default_encoding = sys.getdefaultencoding()
 887         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 888             ["git", "config", "--file",
 889              os.path.join(self.rust_root, ".gitmodules"),
 890              "--get-regexp", "path"]
 891         ).decode(default_encoding).splitlines()]
 892         filtered_submodules = []
 893         submodules_names = []
 894         for module in submodules:
 895             if module.endswith("llvm-project"):
 896                 if self.get_toml('llvm-config') or self.downloading_llvm():
 897                     if self.get_toml('lld') != 'true':
 898                         continue
 899             check = self.check_submodule(module, slow_submodules)
 900             filtered_submodules.append((module, check))
 901             submodules_names.append(module)
 902         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 903                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 904         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 905         recorded_submodules = {}
 906         for data in recorded:
 907             data = data.split()
 908             recorded_submodules[data[3]] = data[2]
 909         for module in filtered_submodules:
 910             self.update_submodule(module[0], module[1], recorded_submodules)
 911         print("Submodules updated in %.2f seconds" % (time() - start_time))
 912
 913     def set_normal_environment(self):
 914         """Set download URL for normal environment"""
 915         if 'RUSTUP_DIST_SERVER' in os.environ:
 916             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 917         else:
 918             self._download_url = 'https://static.rust-lang.org'
 919
 920     def set_dev_environment(self):
 921         """Set download URL for development environment"""
 922         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 923             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 924         else:
 925             self._download_url = 'https://dev-static.rust-lang.org'
 926
 927     def check_vendored_status(self):
 928         """Check that vendoring is configured properly"""
 929         vendor_dir = os.path.join(self.rust_root, 'vendor')
 930         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 931             if os.environ.get('USER') != os.environ['SUDO_USER']:
 932                 self.use_vendored_sources = True
 933                 print('info: looks like you are running this command under `sudo`')
 934                 print('      and so in order to preserve your $HOME this will now')
 935                 print('      use vendored sources by default.')
 936                 if not os.path.exists(vendor_dir):
 937                     print('error: vendoring required, but vendor directory does not exist.')
 938                     print('       Run `cargo vendor` without sudo to initialize the '
 939                           'vendor directory.')
 940                     raise Exception("{} not found".format(vendor_dir))
 941
 942         if self.use_vendored_sources:
 943             if not os.path.exists('.cargo'):
 944                 os.makedirs('.cargo')
 945             with output('.cargo/config') as cargo_config:
 946                 cargo_config.write(
 947                     "[source.crates-io]\n"
 948                     "replace-with = 'vendored-sources'\n"
 949                     "registry = 'https://example.com'\n"
 950                     "\n"
 951                     "[source.vendored-sources]\n"
 952                     "directory = '{}/vendor'\n"
 953                     .format(self.rust_root))
 954         else:
 955             if os.path.exists('.cargo'):
 956                 shutil.rmtree('.cargo')
 957
 958     def ensure_vendored(self):
 959         """Ensure that the vendored sources are available if needed"""
 960         vendor_dir = os.path.join(self.rust_root, 'vendor')
 961         # Note that this does not handle updating the vendored dependencies if
 962         # the rust git repository is updated. Normal development usually does
 963         # not use vendoring, so hopefully this isn't too much of a problem.
 964         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 965             run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
 966                 verbose=self.verbose, cwd=self.rust_root)
 967
 968
 969 def bootstrap(help_triggered):
 970     """Configure, fetch, build and run the initial bootstrap"""
 971
 972     # If the user is asking for help, let them know that the whole download-and-build
 973     # process has to happen before anything is printed out.
 974     if help_triggered:
 975         print("info: Downloading and building bootstrap before processing --help")
 976         print("      command. See src/bootstrap/README.md for help with common")
 977         print("      commands.")
 978
 979     parser = argparse.ArgumentParser(description='Build rust')
 980     parser.add_argument('--config')
 981     parser.add_argument('--build')
 982     parser.add_argument('--clean', action='store_true')
 983     parser.add_argument('-v', '--verbose', action='count', default=0)
 984
 985     args = [a for a in sys.argv if a != '-h' and a != '--help']
 986     args, _ = parser.parse_known_args(args)
 987
 988     # Configure initial bootstrap
 989     build = RustBuild()
 990     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 991     build.verbose = args.verbose
 992     build.clean = args.clean
 993
 994     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
 995     # exists).
 996     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
 997     if not toml_path and os.path.exists('config.toml'):
 998         toml_path = 'config.toml'
 999
1000     if toml_path:
1001         if not os.path.exists(toml_path):
1002             toml_path = os.path.join(build.rust_root, toml_path)
1003
1004         with open(toml_path) as config:
1005             build.config_toml = config.read()
1006
1007     profile = build.get_toml('profile')
1008     if profile is not None:
1009         include_file = 'config.{}.toml'.format(profile)
1010         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1011         include_path = os.path.join(include_dir, include_file)
1012         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1013         # specific key, so appending our defaults at the end allows the user to override them
1014         with open(include_path) as included_toml:
1015             build.config_toml += os.linesep + included_toml.read()
1016
1017     config_verbose = build.get_toml('verbose', 'build')
1018     if config_verbose is not None:
1019         build.verbose = max(build.verbose, int(config_verbose))
1020
1021     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1022
1023     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1024
1025     build.check_vendored_status()
1026
1027     build_dir = build.get_toml('build-dir', 'build') or 'build'
1028     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1029
1030     data = stage0_data(build.rust_root)
1031     build.date = data['date']
1032     build.rustc_channel = data['rustc']
1033     build.cargo_channel = data['cargo']
1034
1035     if "rustfmt" in data:
1036         build.rustfmt_channel = data['rustfmt']
1037
1038     if 'dev' in data:
1039         build.set_dev_environment()
1040     else:
1041         build.set_normal_environment()
1042
1043     build.update_submodules()
1044
1045     # Fetch/build the bootstrap
1046     build.build = args.build or build.build_triple()
1047     build.download_stage0()
1048     sys.stdout.flush()
1049     build.ensure_vendored()
1050     build.build_bootstrap()
1051     sys.stdout.flush()
1052
1053     # Run the bootstrap
1054     args = [build.bootstrap_binary()]
1055     args.extend(sys.argv[1:])
1056     env = os.environ.copy()
1057     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1058     env["BOOTSTRAP_PYTHON"] = sys.executable
1059     env["BUILD_DIR"] = build.build_dir
1060     env["RUSTC_BOOTSTRAP"] = '1'
1061     if toml_path:
1062         env["BOOTSTRAP_CONFIG"] = toml_path
1063     run(args, env=env, verbose=build.verbose)
1064
1065
1066 def main():
1067     """Entry point for the bootstrap process"""
1068     start_time = time()
1069
1070     # x.py help <cmd> ...
1071     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1072         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1073
1074     help_triggered = (
1075         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1076     try:
1077         bootstrap(help_triggered)
1078         if not help_triggered:
1079             print("Build completed successfully in {}".format(
1080                 format_build_time(time() - start_time)))
1081     except (SystemExit, KeyboardInterrupt) as error:
1082         if hasattr(error, 'code') and isinstance(error.code, int):
1083             exit_code = error.code
1084         else:
1085             exit_code = 1
1086             print(error)
1087         if not help_triggered:
1088             print("Build completed unsuccessfully in {}".format(
1089                 format_build_time(time() - start_time)))
1090         sys.exit(exit_code)
1091
1092
1093 if __name__ == '__main__':
1094     main()