src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple():
 191     """Build triple as in LLVM"""
 192     default_encoding = sys.getdefaultencoding()
 193     required = sys.platform != 'win32'
 194     ostype = require(["uname", "-s"], exit=required)
 195     cputype = require(['uname', '-m'], exit=required)
 196
 197     # If we do not have `uname`, assume Windows.
 198     if ostype is None or cputype is None:
 199         return 'x86_64-pc-windows-msvc'
 200
 201     ostype = ostype.decode(default_encoding)
 202     cputype = cputype.decode(default_encoding)
 203
 204     # The goal here is to come up with the same triple as LLVM would,
 205     # at least for the subset of platforms we're willing to target.
 206     ostype_mapper = {
 207         'Darwin': 'apple-darwin',
 208         'DragonFly': 'unknown-dragonfly',
 209         'FreeBSD': 'unknown-freebsd',
 210         'Haiku': 'unknown-haiku',
 211         'NetBSD': 'unknown-netbsd',
 212         'OpenBSD': 'unknown-openbsd'
 213     }
 214
 215     # Consider the direct transformation first and then the special cases
 216     if ostype in ostype_mapper:
 217         ostype = ostype_mapper[ostype]
 218     elif ostype == 'Linux':
 219         os_from_sp = subprocess.check_output(
 220             ['uname', '-o']).strip().decode(default_encoding)
 221         if os_from_sp == 'Android':
 222             ostype = 'linux-android'
 223         else:
 224             ostype = 'unknown-linux-gnu'
 225     elif ostype == 'SunOS':
 226         ostype = 'sun-solaris'
 227         # On Solaris, uname -m will return a machine classification instead
 228         # of a cpu type, so uname -p is recommended instead.  However, the
 229         # output from that option is too generic for our purposes (it will
 230         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 231         # must be used instead.
 232         cputype = require(['isainfo', '-k']).decode(default_encoding)
 233     elif ostype.startswith('MINGW'):
 234         # msys' `uname` does not print gcc configuration, but prints msys
 235         # configuration. so we cannot believe `uname -m`:
 236         # msys1 is always i686 and msys2 is always x86_64.
 237         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 238         # MINGW64 on x86_64.
 239         ostype = 'pc-windows-gnu'
 240         cputype = 'i686'
 241         if os.environ.get('MSYSTEM') == 'MINGW64':
 242             cputype = 'x86_64'
 243     elif ostype.startswith('MSYS'):
 244         ostype = 'pc-windows-gnu'
 245     elif ostype.startswith('CYGWIN_NT'):
 246         cputype = 'i686'
 247         if ostype.endswith('WOW64'):
 248             cputype = 'x86_64'
 249         ostype = 'pc-windows-gnu'
 250     elif sys.platform == 'win32':
 251         # Some Windows platforms might have a `uname` command that returns a
 252         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 253         # these cases, fall back to using sys.platform.
 254         return 'x86_64-pc-windows-msvc'
 255     else:
 256         err = "unknown OS type: {}".format(ostype)
 257         sys.exit(err)
 258
 259     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 260         cputype = subprocess.check_output(
 261               ['uname', '-p']).strip().decode(default_encoding)
 262     cputype_mapper = {
 263         'BePC': 'i686',
 264         'aarch64': 'aarch64',
 265         'amd64': 'x86_64',
 266         'arm64': 'aarch64',
 267         'i386': 'i686',
 268         'i486': 'i686',
 269         'i686': 'i686',
 270         'i786': 'i686',
 271         'powerpc': 'powerpc',
 272         'powerpc64': 'powerpc64',
 273         'powerpc64le': 'powerpc64le',
 274         'ppc': 'powerpc',
 275         'ppc64': 'powerpc64',
 276         'ppc64le': 'powerpc64le',
 277         's390x': 's390x',
 278         'x64': 'x86_64',
 279         'x86': 'i686',
 280         'x86-64': 'x86_64',
 281         'x86_64': 'x86_64'
 282     }
 283
 284     # Consider the direct transformation first and then the special cases
 285     if cputype in cputype_mapper:
 286         cputype = cputype_mapper[cputype]
 287     elif cputype in {'xscale', 'arm'}:
 288         cputype = 'arm'
 289         if ostype == 'linux-android':
 290             ostype = 'linux-androideabi'
 291         elif ostype == 'unknown-freebsd':
 292             cputype = subprocess.check_output(
 293                 ['uname', '-p']).strip().decode(default_encoding)
 294             ostype = 'unknown-freebsd'
 295     elif cputype == 'armv6l':
 296         cputype = 'arm'
 297         if ostype == 'linux-android':
 298             ostype = 'linux-androideabi'
 299         else:
 300             ostype += 'eabihf'
 301     elif cputype in {'armv7l', 'armv8l'}:
 302         cputype = 'armv7'
 303         if ostype == 'linux-android':
 304             ostype = 'linux-androideabi'
 305         else:
 306             ostype += 'eabihf'
 307     elif cputype == 'mips':
 308         if sys.byteorder == 'big':
 309             cputype = 'mips'
 310         elif sys.byteorder == 'little':
 311             cputype = 'mipsel'
 312         else:
 313             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 314     elif cputype == 'mips64':
 315         if sys.byteorder == 'big':
 316             cputype = 'mips64'
 317         elif sys.byteorder == 'little':
 318             cputype = 'mips64el'
 319         else:
 320             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 321         # only the n64 ABI is supported, indicate it
 322         ostype += 'abi64'
 323     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 324         pass
 325     else:
 326         err = "unknown cpu type: {}".format(cputype)
 327         sys.exit(err)
 328
 329     return "{}-{}".format(cputype, ostype)
 330
 331
 332 @contextlib.contextmanager
 333 def output(filepath):
 334     tmp = filepath + '.tmp'
 335     with open(tmp, 'w') as f:
 336         yield f
 337     try:
 338         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 339         os.rename(tmp, filepath)
 340     except OSError:
 341         shutil.copy2(tmp, filepath)
 342         os.remove(tmp)
 343
 344
 345 class RustBuild(object):
 346     """Provide all the methods required to build Rust"""
 347     def __init__(self):
 348         self.cargo_channel = ''
 349         self.date = ''
 350         self._download_url = ''
 351         self.rustc_channel = ''
 352         self.rustfmt_channel = ''
 353         self.build = ''
 354         self.build_dir = ''
 355         self.clean = False
 356         self.config_toml = ''
 357         self.rust_root = ''
 358         self.use_locked_deps = ''
 359         self.use_vendored_sources = ''
 360         self.verbose = False
 361         self.git_version = None
 362         self.nix_deps_dir = None
 363
 364     def download_stage0(self):
 365         """Fetch the build system for Rust, written in Rust
 366
 367         This method will build a cache directory, then it will fetch the
 368         tarball which has the stage0 compiler used to then bootstrap the Rust
 369         compiler itself.
 370
 371         Each downloaded tarball is extracted, after that, the script
 372         will move all the content to the right place.
 373         """
 374         rustc_channel = self.rustc_channel
 375         cargo_channel = self.cargo_channel
 376         rustfmt_channel = self.rustfmt_channel
 377
 378         if self.rustc().startswith(self.bin_root()) and \
 379                 (not os.path.exists(self.rustc()) or
 380                  self.program_out_of_date(self.rustc_stamp())):
 381             if os.path.exists(self.bin_root()):
 382                 shutil.rmtree(self.bin_root())
 383             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 384             filename = "rust-std-{}-{}{}".format(
 385                 rustc_channel, self.build, tarball_suffix)
 386             pattern = "rust-std-{}".format(self.build)
 387             self._download_stage0_helper(filename, pattern, tarball_suffix)
 388
 389             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 392             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 393             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 394             lib_dir = "{}/lib".format(self.bin_root())
 395             for lib in os.listdir(lib_dir):
 396                 if lib.endswith(".so"):
 397                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 398             with output(self.rustc_stamp()) as rust_stamp:
 399                 rust_stamp.write(self.date)
 400
 401         if self.cargo().startswith(self.bin_root()) and \
 402                 (not os.path.exists(self.cargo()) or
 403                  self.program_out_of_date(self.cargo_stamp())):
 404             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 405             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 406                                               tarball_suffix)
 407             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 408             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 409             with output(self.cargo_stamp()) as cargo_stamp:
 410                 cargo_stamp.write(self.date)
 411
 412         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 413             not os.path.exists(self.rustfmt())
 414             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 415         ):
 416             if rustfmt_channel:
 417                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 418                 [channel, date] = rustfmt_channel.split('-', 1)
 419                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 420                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 421                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 422                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 423                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 424                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 425
 426         if self.downloading_llvm():
 427             llvm_sha = subprocess.check_output(["git", "log", "--author=bors",
 428                 "--format=%H", "-n1"]).decode(sys.getdefaultencoding()).strip()
 429             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 430             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 431                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 432                 for binary in ["llvm-config", "FileCheck"]:
 433                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 434                 with output(self.llvm_stamp()) as llvm_stamp:
 435                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 436
 437     def downloading_llvm(self):
 438         opt = self.get_toml('download-ci-llvm', 'llvm')
 439         return opt == "true"
 440
 441     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 442         if date is None:
 443             date = self.date
 444         cache_dst = os.path.join(self.build_dir, "cache")
 445         rustc_cache = os.path.join(cache_dst, date)
 446         if not os.path.exists(rustc_cache):
 447             os.makedirs(rustc_cache)
 448
 449         url = "{}/dist/{}".format(self._download_url, date)
 450         tarball = os.path.join(rustc_cache, filename)
 451         if not os.path.exists(tarball):
 452             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 453         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 454
 455     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 456         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 457         cache_dst = os.path.join(self.build_dir, "cache")
 458         rustc_cache = os.path.join(cache_dst, cache_prefix)
 459         if not os.path.exists(rustc_cache):
 460             os.makedirs(rustc_cache)
 461
 462         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 463         if llvm_assertions:
 464             url = url.replace('rustc-builds', 'rustc-builds-alt')
 465         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 466         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 467         tarball = os.path.join(rustc_cache, filename)
 468         if not os.path.exists(tarball):
 469             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 470         unpack(tarball, tarball_suffix, self.llvm_root(),
 471                 match="rust-dev",
 472                 verbose=self.verbose)
 473
 474     def fix_bin_or_dylib(self, fname):
 475         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 476         or the RPATH section, to fix the dynamic library search path
 477
 478         This method is only required on NixOS and uses the PatchELF utility to
 479         change the interpreter/RPATH of ELF executables.
 480
 481         Please see https://nixos.org/patchelf.html for more information
 482         """
 483         default_encoding = sys.getdefaultencoding()
 484         try:
 485             ostype = subprocess.check_output(
 486                 ['uname', '-s']).strip().decode(default_encoding)
 487         except subprocess.CalledProcessError:
 488             return
 489         except OSError as reason:
 490             if getattr(reason, 'winerror', None) is not None:
 491                 return
 492             raise reason
 493
 494         if ostype != "Linux":
 495             return
 496
 497         if not os.path.exists("/etc/NIXOS"):
 498             return
 499         if os.path.exists("/lib"):
 500             return
 501
 502         # At this point we're pretty sure the user is running NixOS
 503         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 504         print(nix_os_msg, fname)
 505
 506         # Only build `stage0/.nix-deps` once.
 507         nix_deps_dir = self.nix_deps_dir
 508         if not nix_deps_dir:
 509             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 510             if not os.path.exists(nix_deps_dir):
 511                 os.makedirs(nix_deps_dir)
 512
 513             nix_deps = [
 514                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 515                 "stdenv.cc.bintools",
 516
 517                 # Needed as a system dependency of `libLLVM-*.so`.
 518                 "zlib",
 519
 520                 # Needed for patching ELF binaries (see doc comment above).
 521                 "patchelf",
 522             ]
 523
 524             # Run `nix-build` to "build" each dependency (which will likely reuse
 525             # the existing `/nix/store` copy, or at most download a pre-built copy).
 526             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 527             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 528             # ensuring garbage collection will never remove the `/nix/store` path
 529             # (which would break our patched binaries that hardcode those paths).
 530             for dep in nix_deps:
 531                 try:
 532                     subprocess.check_output([
 533                         "nix-build", "<nixpkgs>",
 534                         "-A", dep,
 535                         "-o", "{}/{}".format(nix_deps_dir, dep),
 536                     ])
 537                 except subprocess.CalledProcessError as reason:
 538                     print("warning: failed to call nix-build:", reason)
 539                     return
 540
 541             self.nix_deps_dir = nix_deps_dir
 542
 543         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 544
 545         if fname.endswith(".so"):
 546             # Dynamic library, patch RPATH to point to system dependencies.
 547             dylib_deps = ["zlib"]
 548             rpath_entries = [
 549                 # Relative default, all binary and dynamic libraries we ship
 550                 # appear to have this (even when `../lib` is redundant).
 551                 "$ORIGIN/../lib",
 552             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 553             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 554         else:
 555             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 556             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 557                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 558
 559         try:
 560             subprocess.check_output([patchelf] + patchelf_args + [fname])
 561         except subprocess.CalledProcessError as reason:
 562             print("warning: failed to call patchelf:", reason)
 563             return
 564
 565     def rustc_stamp(self):
 566         """Return the path for .rustc-stamp
 567
 568         >>> rb = RustBuild()
 569         >>> rb.build_dir = "build"
 570         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 571         True
 572         """
 573         return os.path.join(self.bin_root(), '.rustc-stamp')
 574
 575     def cargo_stamp(self):
 576         """Return the path for .cargo-stamp
 577
 578         >>> rb = RustBuild()
 579         >>> rb.build_dir = "build"
 580         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 581         True
 582         """
 583         return os.path.join(self.bin_root(), '.cargo-stamp')
 584
 585     def rustfmt_stamp(self):
 586         """Return the path for .rustfmt-stamp
 587
 588         >>> rb = RustBuild()
 589         >>> rb.build_dir = "build"
 590         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 591         True
 592         """
 593         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 594
 595     def llvm_stamp(self):
 596         """Return the path for .rustfmt-stamp
 597
 598         >>> rb = RustBuild()
 599         >>> rb.build_dir = "build"
 600         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 601         True
 602         """
 603         return os.path.join(self.llvm_root(), '.llvm-stamp')
 604
 605
 606     def program_out_of_date(self, stamp_path, extra=""):
 607         """Check if the given program stamp is out of date"""
 608         if not os.path.exists(stamp_path) or self.clean:
 609             return True
 610         with open(stamp_path, 'r') as stamp:
 611             return (self.date + extra) != stamp.read()
 612
 613     def bin_root(self):
 614         """Return the binary root directory
 615
 616         >>> rb = RustBuild()
 617         >>> rb.build_dir = "build"
 618         >>> rb.bin_root() == os.path.join("build", "stage0")
 619         True
 620
 621         When the 'build' property is given should be a nested directory:
 622
 623         >>> rb.build = "devel"
 624         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 625         True
 626         """
 627         return os.path.join(self.build_dir, self.build, "stage0")
 628
 629     def llvm_root(self):
 630         """Return the CI LLVM root directory
 631
 632         >>> rb = RustBuild()
 633         >>> rb.build_dir = "build"
 634         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 635         True
 636
 637         When the 'build' property is given should be a nested directory:
 638
 639         >>> rb.build = "devel"
 640         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 641         True
 642         """
 643         return os.path.join(self.build_dir, self.build, "ci-llvm")
 644
 645     def get_toml(self, key, section=None):
 646         """Returns the value of the given key in config.toml, otherwise returns None
 647
 648         >>> rb = RustBuild()
 649         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 650         >>> rb.get_toml("key2")
 651         'value2'
 652
 653         If the key does not exists, the result is None:
 654
 655         >>> rb.get_toml("key3") is None
 656         True
 657
 658         Optionally also matches the section the key appears in
 659
 660         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 661         >>> rb.get_toml('key', 'a')
 662         'value1'
 663         >>> rb.get_toml('key', 'b')
 664         'value2'
 665         >>> rb.get_toml('key', 'c') is None
 666         True
 667
 668         >>> rb.config_toml = 'key1 = true'
 669         >>> rb.get_toml("key1")
 670         'true'
 671         """
 672
 673         cur_section = None
 674         for line in self.config_toml.splitlines():
 675             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 676             if section_match is not None:
 677                 cur_section = section_match.group(1)
 678
 679             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 680             if match is not None:
 681                 value = match.group(1)
 682                 if section is None or section == cur_section:
 683                     return self.get_string(value) or value.strip()
 684         return None
 685
 686     def cargo(self):
 687         """Return config path for cargo"""
 688         return self.program_config('cargo')
 689
 690     def rustc(self):
 691         """Return config path for rustc"""
 692         return self.program_config('rustc')
 693
 694     def rustfmt(self):
 695         """Return config path for rustfmt"""
 696         if not self.rustfmt_channel:
 697             return None
 698         return self.program_config('rustfmt')
 699
 700     def program_config(self, program):
 701         """Return config path for the given program
 702
 703         >>> rb = RustBuild()
 704         >>> rb.config_toml = 'rustc = "rustc"\\n'
 705         >>> rb.program_config('rustc')
 706         'rustc'
 707         >>> rb.config_toml = ''
 708         >>> cargo_path = rb.program_config('cargo')
 709         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 710         ... "bin", "cargo")
 711         True
 712         """
 713         config = self.get_toml(program)
 714         if config:
 715             return os.path.expanduser(config)
 716         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 717             program, self.exe_suffix()))
 718
 719     @staticmethod
 720     def get_string(line):
 721         """Return the value between double quotes
 722
 723         >>> RustBuild.get_string('    "devel"   ')
 724         'devel'
 725         >>> RustBuild.get_string("    'devel'   ")
 726         'devel'
 727         >>> RustBuild.get_string('devel') is None
 728         True
 729         >>> RustBuild.get_string('    "devel   ')
 730         ''
 731         """
 732         start = line.find('"')
 733         if start != -1:
 734             end = start + 1 + line[start + 1:].find('"')
 735             return line[start + 1:end]
 736         start = line.find('\'')
 737         if start != -1:
 738             end = start + 1 + line[start + 1:].find('\'')
 739             return line[start + 1:end]
 740         return None
 741
 742     @staticmethod
 743     def exe_suffix():
 744         """Return a suffix for executables"""
 745         if sys.platform == 'win32':
 746             return '.exe'
 747         return ''
 748
 749     def bootstrap_binary(self):
 750         """Return the path of the bootstrap binary
 751
 752         >>> rb = RustBuild()
 753         >>> rb.build_dir = "build"
 754         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 755         ... "debug", "bootstrap")
 756         True
 757         """
 758         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 759
 760     def build_bootstrap(self):
 761         """Build bootstrap"""
 762         build_dir = os.path.join(self.build_dir, "bootstrap")
 763         if self.clean and os.path.exists(build_dir):
 764             shutil.rmtree(build_dir)
 765         env = os.environ.copy()
 766         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 767         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 768         if "CARGO_BUILD_TARGET" in env:
 769             del env["CARGO_BUILD_TARGET"]
 770         env["CARGO_TARGET_DIR"] = build_dir
 771         env["RUSTC"] = self.rustc()
 772         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 773             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 774             if "LD_LIBRARY_PATH" in env else ""
 775         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 776             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 777             if "DYLD_LIBRARY_PATH" in env else ""
 778         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 779             (os.pathsep + env["LIBRARY_PATH"]) \
 780             if "LIBRARY_PATH" in env else ""
 781         # preserve existing RUSTFLAGS
 782         env.setdefault("RUSTFLAGS", "")
 783         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 784
 785         build_section = "target.{}".format(self.build_triple())
 786         target_features = []
 787         if self.get_toml("crt-static", build_section) == "true":
 788             target_features += ["+crt-static"]
 789         elif self.get_toml("crt-static", build_section) == "false":
 790             target_features += ["-crt-static"]
 791         if target_features:
 792             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 793         target_linker = self.get_toml("linker", build_section)
 794         if target_linker is not None:
 795             env["RUSTFLAGS"] += " -C linker=" + target_linker
 796         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 797         if self.get_toml("deny-warnings", "rust") != "false":
 798             env["RUSTFLAGS"] += " -Dwarnings"
 799
 800         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 801             os.pathsep + env["PATH"]
 802         if not os.path.isfile(self.cargo()):
 803             raise Exception("no cargo executable found at `{}`".format(
 804                 self.cargo()))
 805         args = [self.cargo(), "build", "--manifest-path",
 806                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 807         for _ in range(1, self.verbose):
 808             args.append("--verbose")
 809         if self.use_locked_deps:
 810             args.append("--locked")
 811         if self.use_vendored_sources:
 812             args.append("--frozen")
 813         run(args, env=env, verbose=self.verbose)
 814
 815     def build_triple(self):
 816         """Build triple as in LLVM"""
 817         config = self.get_toml('build')
 818         if config:
 819             return config
 820         return default_build_triple()
 821
 822     def check_submodule(self, module, slow_submodules):
 823         if not slow_submodules:
 824             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 825                                            cwd=os.path.join(self.rust_root, module),
 826                                            stdout=subprocess.PIPE)
 827             return checked_out
 828         else:
 829             return None
 830
 831     def update_submodule(self, module, checked_out, recorded_submodules):
 832         module_path = os.path.join(self.rust_root, module)
 833
 834         if checked_out is not None:
 835             default_encoding = sys.getdefaultencoding()
 836             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 837             if recorded_submodules[module] == checked_out:
 838                 return
 839
 840         print("Updating submodule", module)
 841
 842         run(["git", "submodule", "-q", "sync", module],
 843             cwd=self.rust_root, verbose=self.verbose)
 844
 845         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 846         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 847             update_args.append("--progress")
 848         update_args.append(module)
 849         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 850
 851         run(["git", "reset", "-q", "--hard"],
 852             cwd=module_path, verbose=self.verbose)
 853         run(["git", "clean", "-qdfx"],
 854             cwd=module_path, verbose=self.verbose)
 855
 856     def update_submodules(self):
 857         """Update submodules"""
 858         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 859                 self.get_toml('submodules') == "false":
 860             return
 861
 862         default_encoding = sys.getdefaultencoding()
 863
 864         # check the existence and version of 'git' command
 865         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 866         self.git_version = distutils.version.LooseVersion(git_version_str)
 867
 868         slow_submodules = self.get_toml('fast-submodules') == "false"
 869         start_time = time()
 870         if slow_submodules:
 871             print('Unconditionally updating all submodules')
 872         else:
 873             print('Updating only changed submodules')
 874         default_encoding = sys.getdefaultencoding()
 875         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 876             ["git", "config", "--file",
 877              os.path.join(self.rust_root, ".gitmodules"),
 878              "--get-regexp", "path"]
 879         ).decode(default_encoding).splitlines()]
 880         filtered_submodules = []
 881         submodules_names = []
 882         for module in submodules:
 883             if module.endswith("llvm-project"):
 884                 if self.get_toml('llvm-config') or self.get_toml('download-ci-llvm') == 'true':
 885                     if self.get_toml('lld') != 'true':
 886                         continue
 887             check = self.check_submodule(module, slow_submodules)
 888             filtered_submodules.append((module, check))
 889             submodules_names.append(module)
 890         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 891                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 892         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 893         recorded_submodules = {}
 894         for data in recorded:
 895             data = data.split()
 896             recorded_submodules[data[3]] = data[2]
 897         for module in filtered_submodules:
 898             self.update_submodule(module[0], module[1], recorded_submodules)
 899         print("Submodules updated in %.2f seconds" % (time() - start_time))
 900
 901     def set_normal_environment(self):
 902         """Set download URL for normal environment"""
 903         if 'RUSTUP_DIST_SERVER' in os.environ:
 904             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 905         else:
 906             self._download_url = 'https://static.rust-lang.org'
 907
 908     def set_dev_environment(self):
 909         """Set download URL for development environment"""
 910         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 911             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 912         else:
 913             self._download_url = 'https://dev-static.rust-lang.org'
 914
 915     def check_vendored_status(self):
 916         """Check that vendoring is configured properly"""
 917         vendor_dir = os.path.join(self.rust_root, 'vendor')
 918         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 919             if os.environ.get('USER') != os.environ['SUDO_USER']:
 920                 self.use_vendored_sources = True
 921                 print('info: looks like you are running this command under `sudo`')
 922                 print('      and so in order to preserve your $HOME this will now')
 923                 print('      use vendored sources by default.')
 924                 if not os.path.exists(vendor_dir):
 925                     print('error: vendoring required, but vendor directory does not exist.')
 926                     print('       Run `cargo vendor` without sudo to initialize the '
 927                           'vendor directory.')
 928                     raise Exception("{} not found".format(vendor_dir))
 929
 930         if self.use_vendored_sources:
 931             if not os.path.exists('.cargo'):
 932                 os.makedirs('.cargo')
 933             with output('.cargo/config') as cargo_config:
 934                 cargo_config.write(
 935                     "[source.crates-io]\n"
 936                     "replace-with = 'vendored-sources'\n"
 937                     "registry = 'https://example.com'\n"
 938                     "\n"
 939                     "[source.vendored-sources]\n"
 940                     "directory = '{}/vendor'\n"
 941                     .format(self.rust_root))
 942         else:
 943             if os.path.exists('.cargo'):
 944                 shutil.rmtree('.cargo')
 945
 946     def ensure_vendored(self):
 947         """Ensure that the vendored sources are available if needed"""
 948         vendor_dir = os.path.join(self.rust_root, 'vendor')
 949         # Note that this does not handle updating the vendored dependencies if
 950         # the rust git repository is updated. Normal development usually does
 951         # not use vendoring, so hopefully this isn't too much of a problem.
 952         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 953             run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
 954                 verbose=self.verbose, cwd=self.rust_root)
 955
 956
 957 def bootstrap(help_triggered):
 958     """Configure, fetch, build and run the initial bootstrap"""
 959
 960     # If the user is asking for help, let them know that the whole download-and-build
 961     # process has to happen before anything is printed out.
 962     if help_triggered:
 963         print("info: Downloading and building bootstrap before processing --help")
 964         print("      command. See src/bootstrap/README.md for help with common")
 965         print("      commands.")
 966
 967     parser = argparse.ArgumentParser(description='Build rust')
 968     parser.add_argument('--config')
 969     parser.add_argument('--build')
 970     parser.add_argument('--clean', action='store_true')
 971     parser.add_argument('-v', '--verbose', action='count', default=0)
 972
 973     args = [a for a in sys.argv if a != '-h' and a != '--help']
 974     args, _ = parser.parse_known_args(args)
 975
 976     # Configure initial bootstrap
 977     build = RustBuild()
 978     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 979     build.verbose = args.verbose
 980     build.clean = args.clean
 981
 982     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
 983     # exists).
 984     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
 985     if not toml_path and os.path.exists('config.toml'):
 986         toml_path = 'config.toml'
 987
 988     if toml_path:
 989         if not os.path.exists(toml_path):
 990             toml_path = os.path.join(build.rust_root, toml_path)
 991
 992         with open(toml_path) as config:
 993             build.config_toml = config.read()
 994
 995     config_verbose = build.get_toml('verbose', 'build')
 996     if config_verbose is not None:
 997         build.verbose = max(build.verbose, int(config_verbose))
 998
 999     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1000
1001     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1002
1003     build.check_vendored_status()
1004
1005     build_dir = build.get_toml('build-dir', 'build') or 'build'
1006     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1007
1008     data = stage0_data(build.rust_root)
1009     build.date = data['date']
1010     build.rustc_channel = data['rustc']
1011     build.cargo_channel = data['cargo']
1012
1013     if "rustfmt" in data:
1014         build.rustfmt_channel = data['rustfmt']
1015
1016     if 'dev' in data:
1017         build.set_dev_environment()
1018     else:
1019         build.set_normal_environment()
1020
1021     build.update_submodules()
1022
1023     # Fetch/build the bootstrap
1024     build.build = args.build or build.build_triple()
1025     build.download_stage0()
1026     sys.stdout.flush()
1027     build.ensure_vendored()
1028     build.build_bootstrap()
1029     sys.stdout.flush()
1030
1031     # Run the bootstrap
1032     args = [build.bootstrap_binary()]
1033     args.extend(sys.argv[1:])
1034     env = os.environ.copy()
1035     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1036     env["BOOTSTRAP_PYTHON"] = sys.executable
1037     env["BUILD_DIR"] = build.build_dir
1038     env["RUSTC_BOOTSTRAP"] = '1'
1039     if toml_path:
1040         env["BOOTSTRAP_CONFIG"] = toml_path
1041     run(args, env=env, verbose=build.verbose)
1042
1043
1044 def main():
1045     """Entry point for the bootstrap process"""
1046     start_time = time()
1047
1048     # x.py help <cmd> ...
1049     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1050         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1051
1052     help_triggered = (
1053         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1054     try:
1055         bootstrap(help_triggered)
1056         if not help_triggered:
1057             print("Build completed successfully in {}".format(
1058                 format_build_time(time() - start_time)))
1059     except (SystemExit, KeyboardInterrupt) as error:
1060         if hasattr(error, 'code') and isinstance(error.code, int):
1061             exit_code = error.code
1062         else:
1063             exit_code = 1
1064             print(error)
1065         if not help_triggered:
1066             print("Build completed unsuccessfully in {}".format(
1067                 format_build_time(time() - start_time)))
1068         sys.exit(exit_code)
1069
1070
1071 if __name__ == '__main__':
1072     main()