src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple(verbose):
 191     """Build triple as in LLVM"""
 192     # If the user already has a host build triple with an existing `rustc`
 193     # install, use their preference. This fixes most issues with Windows builds
 194     # being detected as GNU instead of MSVC.
 195     try:
 196         version = subprocess.check_output(["rustc", "--version", "--verbose"])
 197         host = next(x for x in version.split('\n') if x.startswith("host: "))
 198         triple = host.split("host: ")[1]
 199         if verbose:
 200             print("detected default triple {}".format(triple))
 201         return triple
 202     except Exception as e:
 203         if verbose:
 204             print("rustup not detected: {}".format(e))
 205             print("falling back to auto-detect")
 206
 207     default_encoding = sys.getdefaultencoding()
 208     required = sys.platform != 'win32'
 209     ostype = require(["uname", "-s"], exit=required)
 210     cputype = require(['uname', '-m'], exit=required)
 211
 212     # If we do not have `uname`, assume Windows.
 213     if ostype is None or cputype is None:
 214         return 'x86_64-pc-windows-msvc'
 215
 216     ostype = ostype.decode(default_encoding)
 217     cputype = cputype.decode(default_encoding)
 218
 219     # The goal here is to come up with the same triple as LLVM would,
 220     # at least for the subset of platforms we're willing to target.
 221     ostype_mapper = {
 222         'Darwin': 'apple-darwin',
 223         'DragonFly': 'unknown-dragonfly',
 224         'FreeBSD': 'unknown-freebsd',
 225         'Haiku': 'unknown-haiku',
 226         'NetBSD': 'unknown-netbsd',
 227         'OpenBSD': 'unknown-openbsd'
 228     }
 229
 230     # Consider the direct transformation first and then the special cases
 231     if ostype in ostype_mapper:
 232         ostype = ostype_mapper[ostype]
 233     elif ostype == 'Linux':
 234         os_from_sp = subprocess.check_output(
 235             ['uname', '-o']).strip().decode(default_encoding)
 236         if os_from_sp == 'Android':
 237             ostype = 'linux-android'
 238         else:
 239             ostype = 'unknown-linux-gnu'
 240     elif ostype == 'SunOS':
 241         ostype = 'sun-solaris'
 242         # On Solaris, uname -m will return a machine classification instead
 243         # of a cpu type, so uname -p is recommended instead.  However, the
 244         # output from that option is too generic for our purposes (it will
 245         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 246         # must be used instead.
 247         cputype = require(['isainfo', '-k']).decode(default_encoding)
 248     elif ostype.startswith('MINGW'):
 249         # msys' `uname` does not print gcc configuration, but prints msys
 250         # configuration. so we cannot believe `uname -m`:
 251         # msys1 is always i686 and msys2 is always x86_64.
 252         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 253         # MINGW64 on x86_64.
 254         ostype = 'pc-windows-gnu'
 255         cputype = 'i686'
 256         if os.environ.get('MSYSTEM') == 'MINGW64':
 257             cputype = 'x86_64'
 258     elif ostype.startswith('MSYS'):
 259         ostype = 'pc-windows-gnu'
 260     elif ostype.startswith('CYGWIN_NT'):
 261         cputype = 'i686'
 262         if ostype.endswith('WOW64'):
 263             cputype = 'x86_64'
 264         ostype = 'pc-windows-gnu'
 265     elif sys.platform == 'win32':
 266         # Some Windows platforms might have a `uname` command that returns a
 267         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 268         # these cases, fall back to using sys.platform.
 269         return 'x86_64-pc-windows-msvc'
 270     else:
 271         err = "unknown OS type: {}".format(ostype)
 272         sys.exit(err)
 273
 274     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 275         cputype = subprocess.check_output(
 276               ['uname', '-p']).strip().decode(default_encoding)
 277     cputype_mapper = {
 278         'BePC': 'i686',
 279         'aarch64': 'aarch64',
 280         'amd64': 'x86_64',
 281         'arm64': 'aarch64',
 282         'i386': 'i686',
 283         'i486': 'i686',
 284         'i686': 'i686',
 285         'i786': 'i686',
 286         'powerpc': 'powerpc',
 287         'powerpc64': 'powerpc64',
 288         'powerpc64le': 'powerpc64le',
 289         'ppc': 'powerpc',
 290         'ppc64': 'powerpc64',
 291         'ppc64le': 'powerpc64le',
 292         's390x': 's390x',
 293         'x64': 'x86_64',
 294         'x86': 'i686',
 295         'x86-64': 'x86_64',
 296         'x86_64': 'x86_64'
 297     }
 298
 299     # Consider the direct transformation first and then the special cases
 300     if cputype in cputype_mapper:
 301         cputype = cputype_mapper[cputype]
 302     elif cputype in {'xscale', 'arm'}:
 303         cputype = 'arm'
 304         if ostype == 'linux-android':
 305             ostype = 'linux-androideabi'
 306         elif ostype == 'unknown-freebsd':
 307             cputype = subprocess.check_output(
 308                 ['uname', '-p']).strip().decode(default_encoding)
 309             ostype = 'unknown-freebsd'
 310     elif cputype == 'armv6l':
 311         cputype = 'arm'
 312         if ostype == 'linux-android':
 313             ostype = 'linux-androideabi'
 314         else:
 315             ostype += 'eabihf'
 316     elif cputype in {'armv7l', 'armv8l'}:
 317         cputype = 'armv7'
 318         if ostype == 'linux-android':
 319             ostype = 'linux-androideabi'
 320         else:
 321             ostype += 'eabihf'
 322     elif cputype == 'mips':
 323         if sys.byteorder == 'big':
 324             cputype = 'mips'
 325         elif sys.byteorder == 'little':
 326             cputype = 'mipsel'
 327         else:
 328             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 329     elif cputype == 'mips64':
 330         if sys.byteorder == 'big':
 331             cputype = 'mips64'
 332         elif sys.byteorder == 'little':
 333             cputype = 'mips64el'
 334         else:
 335             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 336         # only the n64 ABI is supported, indicate it
 337         ostype += 'abi64'
 338     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 339         pass
 340     else:
 341         err = "unknown cpu type: {}".format(cputype)
 342         sys.exit(err)
 343
 344     return "{}-{}".format(cputype, ostype)
 345
 346
 347 @contextlib.contextmanager
 348 def output(filepath):
 349     tmp = filepath + '.tmp'
 350     with open(tmp, 'w') as f:
 351         yield f
 352     try:
 353         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 354         os.rename(tmp, filepath)
 355     except OSError:
 356         shutil.copy2(tmp, filepath)
 357         os.remove(tmp)
 358
 359
 360 class RustBuild(object):
 361     """Provide all the methods required to build Rust"""
 362     def __init__(self):
 363         self.date = ''
 364         self._download_url = ''
 365         self.rustc_channel = ''
 366         self.rustfmt_channel = ''
 367         self.build = ''
 368         self.build_dir = ''
 369         self.clean = False
 370         self.config_toml = ''
 371         self.rust_root = ''
 372         self.use_locked_deps = ''
 373         self.use_vendored_sources = ''
 374         self.verbose = False
 375         self.git_version = None
 376         self.nix_deps_dir = None
 377
 378     def download_stage0(self):
 379         """Fetch the build system for Rust, written in Rust
 380
 381         This method will build a cache directory, then it will fetch the
 382         tarball which has the stage0 compiler used to then bootstrap the Rust
 383         compiler itself.
 384
 385         Each downloaded tarball is extracted, after that, the script
 386         will move all the content to the right place.
 387         """
 388         rustc_channel = self.rustc_channel
 389         rustfmt_channel = self.rustfmt_channel
 390
 391         if self.rustc().startswith(self.bin_root()) and \
 392                 (not os.path.exists(self.rustc()) or
 393                  self.program_out_of_date(self.rustc_stamp())):
 394             if os.path.exists(self.bin_root()):
 395                 shutil.rmtree(self.bin_root())
 396             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 397             filename = "rust-std-{}-{}{}".format(
 398                 rustc_channel, self.build, tarball_suffix)
 399             pattern = "rust-std-{}".format(self.build)
 400             self._download_stage0_helper(filename, pattern, tarball_suffix)
 401             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 402                                               tarball_suffix)
 403             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 404             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 405                                               tarball_suffix)
 406             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 407             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 408             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 409             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 410             lib_dir = "{}/lib".format(self.bin_root())
 411             for lib in os.listdir(lib_dir):
 412                 if lib.endswith(".so"):
 413                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 414             with output(self.rustc_stamp()) as rust_stamp:
 415                 rust_stamp.write(self.date)
 416
 417         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 418             not os.path.exists(self.rustfmt())
 419             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 420         ):
 421             if rustfmt_channel:
 422                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 423                 [channel, date] = rustfmt_channel.split('-', 1)
 424                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 425                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 426                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 427                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 428                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 429                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 430
 431         if self.downloading_llvm():
 432             # We want the most recent LLVM submodule update to avoid downloading
 433             # LLVM more often than necessary.
 434             #
 435             # This git command finds that commit SHA, looking for bors-authored
 436             # merges that modified src/llvm-project.
 437             #
 438             # This works even in a repository that has not yet initialized
 439             # submodules.
 440             top_level = subprocess.check_output([
 441                 "git", "rev-parse", "--show-toplevel",
 442             ]).decode(sys.getdefaultencoding()).strip()
 443             llvm_sha = subprocess.check_output([
 444                 "git", "log", "--author=bors", "--format=%H", "-n1",
 445                 "-m", "--first-parent",
 446                 "--",
 447                 "{}/src/llvm-project".format(top_level),
 448                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 449             ]).decode(sys.getdefaultencoding()).strip()
 450             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 451             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 452                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 453                 for binary in ["llvm-config", "FileCheck"]:
 454                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 455                 with output(self.llvm_stamp()) as llvm_stamp:
 456                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 457
 458     def downloading_llvm(self):
 459         opt = self.get_toml('download-ci-llvm', 'llvm')
 460         return opt == "true" \
 461             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 462
 463     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 464         if date is None:
 465             date = self.date
 466         cache_dst = os.path.join(self.build_dir, "cache")
 467         rustc_cache = os.path.join(cache_dst, date)
 468         if not os.path.exists(rustc_cache):
 469             os.makedirs(rustc_cache)
 470
 471         url = "{}/dist/{}".format(self._download_url, date)
 472         tarball = os.path.join(rustc_cache, filename)
 473         if not os.path.exists(tarball):
 474             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 475         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 476
 477     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 478         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 479         cache_dst = os.path.join(self.build_dir, "cache")
 480         rustc_cache = os.path.join(cache_dst, cache_prefix)
 481         if not os.path.exists(rustc_cache):
 482             os.makedirs(rustc_cache)
 483
 484         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 485         if llvm_assertions:
 486             url = url.replace('rustc-builds', 'rustc-builds-alt')
 487         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 488         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 489         tarball = os.path.join(rustc_cache, filename)
 490         if not os.path.exists(tarball):
 491             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 492         unpack(tarball, tarball_suffix, self.llvm_root(),
 493                 match="rust-dev",
 494                 verbose=self.verbose)
 495
 496     def fix_bin_or_dylib(self, fname):
 497         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 498         or the RPATH section, to fix the dynamic library search path
 499
 500         This method is only required on NixOS and uses the PatchELF utility to
 501         change the interpreter/RPATH of ELF executables.
 502
 503         Please see https://nixos.org/patchelf.html for more information
 504         """
 505         default_encoding = sys.getdefaultencoding()
 506         try:
 507             ostype = subprocess.check_output(
 508                 ['uname', '-s']).strip().decode(default_encoding)
 509         except subprocess.CalledProcessError:
 510             return
 511         except OSError as reason:
 512             if getattr(reason, 'winerror', None) is not None:
 513                 return
 514             raise reason
 515
 516         if ostype != "Linux":
 517             return
 518
 519         if not os.path.exists("/etc/NIXOS"):
 520             return
 521         if os.path.exists("/lib"):
 522             return
 523
 524         # At this point we're pretty sure the user is running NixOS
 525         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 526         print(nix_os_msg, fname)
 527
 528         # Only build `stage0/.nix-deps` once.
 529         nix_deps_dir = self.nix_deps_dir
 530         if not nix_deps_dir:
 531             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 532             if not os.path.exists(nix_deps_dir):
 533                 os.makedirs(nix_deps_dir)
 534
 535             nix_deps = [
 536                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 537                 "stdenv.cc.bintools",
 538
 539                 # Needed as a system dependency of `libLLVM-*.so`.
 540                 "zlib",
 541
 542                 # Needed for patching ELF binaries (see doc comment above).
 543                 "patchelf",
 544             ]
 545
 546             # Run `nix-build` to "build" each dependency (which will likely reuse
 547             # the existing `/nix/store` copy, or at most download a pre-built copy).
 548             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 549             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 550             # ensuring garbage collection will never remove the `/nix/store` path
 551             # (which would break our patched binaries that hardcode those paths).
 552             for dep in nix_deps:
 553                 try:
 554                     subprocess.check_output([
 555                         "nix-build", "<nixpkgs>",
 556                         "-A", dep,
 557                         "-o", "{}/{}".format(nix_deps_dir, dep),
 558                     ])
 559                 except subprocess.CalledProcessError as reason:
 560                     print("warning: failed to call nix-build:", reason)
 561                     return
 562
 563             self.nix_deps_dir = nix_deps_dir
 564
 565         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 566
 567         if fname.endswith(".so"):
 568             # Dynamic library, patch RPATH to point to system dependencies.
 569             dylib_deps = ["zlib"]
 570             rpath_entries = [
 571                 # Relative default, all binary and dynamic libraries we ship
 572                 # appear to have this (even when `../lib` is redundant).
 573                 "$ORIGIN/../lib",
 574             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 575             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 576         else:
 577             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 578             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 579                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 580
 581         try:
 582             subprocess.check_output([patchelf] + patchelf_args + [fname])
 583         except subprocess.CalledProcessError as reason:
 584             print("warning: failed to call patchelf:", reason)
 585             return
 586
 587     def rustc_stamp(self):
 588         """Return the path for .rustc-stamp
 589
 590         >>> rb = RustBuild()
 591         >>> rb.build_dir = "build"
 592         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 593         True
 594         """
 595         return os.path.join(self.bin_root(), '.rustc-stamp')
 596
 597     def rustfmt_stamp(self):
 598         """Return the path for .rustfmt-stamp
 599
 600         >>> rb = RustBuild()
 601         >>> rb.build_dir = "build"
 602         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 603         True
 604         """
 605         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 606
 607     def llvm_stamp(self):
 608         """Return the path for .rustfmt-stamp
 609
 610         >>> rb = RustBuild()
 611         >>> rb.build_dir = "build"
 612         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 613         True
 614         """
 615         return os.path.join(self.llvm_root(), '.llvm-stamp')
 616
 617
 618     def program_out_of_date(self, stamp_path, extra=""):
 619         """Check if the given program stamp is out of date"""
 620         if not os.path.exists(stamp_path) or self.clean:
 621             return True
 622         with open(stamp_path, 'r') as stamp:
 623             return (self.date + extra) != stamp.read()
 624
 625     def bin_root(self):
 626         """Return the binary root directory
 627
 628         >>> rb = RustBuild()
 629         >>> rb.build_dir = "build"
 630         >>> rb.bin_root() == os.path.join("build", "stage0")
 631         True
 632
 633         When the 'build' property is given should be a nested directory:
 634
 635         >>> rb.build = "devel"
 636         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 637         True
 638         """
 639         return os.path.join(self.build_dir, self.build, "stage0")
 640
 641     def llvm_root(self):
 642         """Return the CI LLVM root directory
 643
 644         >>> rb = RustBuild()
 645         >>> rb.build_dir = "build"
 646         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 647         True
 648
 649         When the 'build' property is given should be a nested directory:
 650
 651         >>> rb.build = "devel"
 652         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 653         True
 654         """
 655         return os.path.join(self.build_dir, self.build, "ci-llvm")
 656
 657     def get_toml(self, key, section=None):
 658         """Returns the value of the given key in config.toml, otherwise returns None
 659
 660         >>> rb = RustBuild()
 661         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 662         >>> rb.get_toml("key2")
 663         'value2'
 664
 665         If the key does not exists, the result is None:
 666
 667         >>> rb.get_toml("key3") is None
 668         True
 669
 670         Optionally also matches the section the key appears in
 671
 672         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 673         >>> rb.get_toml('key', 'a')
 674         'value1'
 675         >>> rb.get_toml('key', 'b')
 676         'value2'
 677         >>> rb.get_toml('key', 'c') is None
 678         True
 679
 680         >>> rb.config_toml = 'key1 = true'
 681         >>> rb.get_toml("key1")
 682         'true'
 683         """
 684
 685         cur_section = None
 686         for line in self.config_toml.splitlines():
 687             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 688             if section_match is not None:
 689                 cur_section = section_match.group(1)
 690
 691             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 692             if match is not None:
 693                 value = match.group(1)
 694                 if section is None or section == cur_section:
 695                     return self.get_string(value) or value.strip()
 696         return None
 697
 698     def cargo(self):
 699         """Return config path for cargo"""
 700         return self.program_config('cargo')
 701
 702     def rustc(self):
 703         """Return config path for rustc"""
 704         return self.program_config('rustc')
 705
 706     def rustfmt(self):
 707         """Return config path for rustfmt"""
 708         if not self.rustfmt_channel:
 709             return None
 710         return self.program_config('rustfmt')
 711
 712     def program_config(self, program):
 713         """Return config path for the given program
 714
 715         >>> rb = RustBuild()
 716         >>> rb.config_toml = 'rustc = "rustc"\\n'
 717         >>> rb.program_config('rustc')
 718         'rustc'
 719         >>> rb.config_toml = ''
 720         >>> cargo_path = rb.program_config('cargo')
 721         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 722         ... "bin", "cargo")
 723         True
 724         """
 725         config = self.get_toml(program)
 726         if config:
 727             return os.path.expanduser(config)
 728         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 729             program, self.exe_suffix()))
 730
 731     @staticmethod
 732     def get_string(line):
 733         """Return the value between double quotes
 734
 735         >>> RustBuild.get_string('    "devel"   ')
 736         'devel'
 737         >>> RustBuild.get_string("    'devel'   ")
 738         'devel'
 739         >>> RustBuild.get_string('devel') is None
 740         True
 741         >>> RustBuild.get_string('    "devel   ')
 742         ''
 743         """
 744         start = line.find('"')
 745         if start != -1:
 746             end = start + 1 + line[start + 1:].find('"')
 747             return line[start + 1:end]
 748         start = line.find('\'')
 749         if start != -1:
 750             end = start + 1 + line[start + 1:].find('\'')
 751             return line[start + 1:end]
 752         return None
 753
 754     @staticmethod
 755     def exe_suffix():
 756         """Return a suffix for executables"""
 757         if sys.platform == 'win32':
 758             return '.exe'
 759         return ''
 760
 761     def bootstrap_binary(self):
 762         """Return the path of the bootstrap binary
 763
 764         >>> rb = RustBuild()
 765         >>> rb.build_dir = "build"
 766         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 767         ... "debug", "bootstrap")
 768         True
 769         """
 770         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 771
 772     def build_bootstrap(self):
 773         """Build bootstrap"""
 774         build_dir = os.path.join(self.build_dir, "bootstrap")
 775         if self.clean and os.path.exists(build_dir):
 776             shutil.rmtree(build_dir)
 777         env = os.environ.copy()
 778         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 779         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 780         if "CARGO_BUILD_TARGET" in env:
 781             del env["CARGO_BUILD_TARGET"]
 782         env["CARGO_TARGET_DIR"] = build_dir
 783         env["RUSTC"] = self.rustc()
 784         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 785             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 786             if "LD_LIBRARY_PATH" in env else ""
 787         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 788             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 789             if "DYLD_LIBRARY_PATH" in env else ""
 790         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 791             (os.pathsep + env["LIBRARY_PATH"]) \
 792             if "LIBRARY_PATH" in env else ""
 793         # preserve existing RUSTFLAGS
 794         env.setdefault("RUSTFLAGS", "")
 795         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 796
 797         build_section = "target.{}".format(self.build_triple())
 798         target_features = []
 799         if self.get_toml("crt-static", build_section) == "true":
 800             target_features += ["+crt-static"]
 801         elif self.get_toml("crt-static", build_section) == "false":
 802             target_features += ["-crt-static"]
 803         if target_features:
 804             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 805         target_linker = self.get_toml("linker", build_section)
 806         if target_linker is not None:
 807             env["RUSTFLAGS"] += " -C linker=" + target_linker
 808         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 809         if self.get_toml("deny-warnings", "rust") != "false":
 810             env["RUSTFLAGS"] += " -Dwarnings"
 811
 812         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 813             os.pathsep + env["PATH"]
 814         if not os.path.isfile(self.cargo()):
 815             raise Exception("no cargo executable found at `{}`".format(
 816                 self.cargo()))
 817         args = [self.cargo(), "build", "--manifest-path",
 818                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 819         for _ in range(1, self.verbose):
 820             args.append("--verbose")
 821         if self.use_locked_deps:
 822             args.append("--locked")
 823         if self.use_vendored_sources:
 824             args.append("--frozen")
 825         run(args, env=env, verbose=self.verbose)
 826
 827     def build_triple(self):
 828         """Build triple as in LLVM"""
 829         config = self.get_toml('build')
 830         if config:
 831             return config
 832         return default_build_triple(self.verbose)
 833
 834     def check_submodule(self, module, slow_submodules):
 835         if not slow_submodules:
 836             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 837                                            cwd=os.path.join(self.rust_root, module),
 838                                            stdout=subprocess.PIPE)
 839             return checked_out
 840         else:
 841             return None
 842
 843     def update_submodule(self, module, checked_out, recorded_submodules):
 844         module_path = os.path.join(self.rust_root, module)
 845
 846         if checked_out is not None:
 847             default_encoding = sys.getdefaultencoding()
 848             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 849             if recorded_submodules[module] == checked_out:
 850                 return
 851
 852         print("Updating submodule", module)
 853
 854         run(["git", "submodule", "-q", "sync", module],
 855             cwd=self.rust_root, verbose=self.verbose)
 856
 857         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 858         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 859             update_args.append("--progress")
 860         update_args.append(module)
 861         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 862
 863         run(["git", "reset", "-q", "--hard"],
 864             cwd=module_path, verbose=self.verbose)
 865         run(["git", "clean", "-qdfx"],
 866             cwd=module_path, verbose=self.verbose)
 867
 868     def update_submodules(self):
 869         """Update submodules"""
 870         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 871                 self.get_toml('submodules') == "false":
 872             return
 873
 874         default_encoding = sys.getdefaultencoding()
 875
 876         # check the existence and version of 'git' command
 877         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 878         self.git_version = distutils.version.LooseVersion(git_version_str)
 879
 880         slow_submodules = self.get_toml('fast-submodules') == "false"
 881         start_time = time()
 882         if slow_submodules:
 883             print('Unconditionally updating all submodules')
 884         else:
 885             print('Updating only changed submodules')
 886         default_encoding = sys.getdefaultencoding()
 887         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 888             ["git", "config", "--file",
 889              os.path.join(self.rust_root, ".gitmodules"),
 890              "--get-regexp", "path"]
 891         ).decode(default_encoding).splitlines()]
 892         filtered_submodules = []
 893         submodules_names = []
 894         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
 895         for module in submodules:
 896             if module.endswith("llvm-project"):
 897                 # Don't sync the llvm-project submodule either if an external LLVM
 898                 # was provided, or if we are downloading LLVM. Also, if the
 899                 # submodule has been initialized already, sync it anyways so that
 900                 # it doesn't mess up contributor pull requests.
 901                 if self.get_toml('llvm-config') or self.downloading_llvm():
 902                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
 903                         continue
 904             check = self.check_submodule(module, slow_submodules)
 905             filtered_submodules.append((module, check))
 906             submodules_names.append(module)
 907         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 908                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 909         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 910         recorded_submodules = {}
 911         for data in recorded:
 912             data = data.split()
 913             recorded_submodules[data[3]] = data[2]
 914         for module in filtered_submodules:
 915             self.update_submodule(module[0], module[1], recorded_submodules)
 916         print("Submodules updated in %.2f seconds" % (time() - start_time))
 917
 918     def set_normal_environment(self):
 919         """Set download URL for normal environment"""
 920         if 'RUSTUP_DIST_SERVER' in os.environ:
 921             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 922         else:
 923             self._download_url = 'https://static.rust-lang.org'
 924
 925     def set_dev_environment(self):
 926         """Set download URL for development environment"""
 927         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 928             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 929         else:
 930             self._download_url = 'https://dev-static.rust-lang.org'
 931
 932     def check_vendored_status(self):
 933         """Check that vendoring is configured properly"""
 934         vendor_dir = os.path.join(self.rust_root, 'vendor')
 935         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 936             if os.environ.get('USER') != os.environ['SUDO_USER']:
 937                 self.use_vendored_sources = True
 938                 print('info: looks like you are running this command under `sudo`')
 939                 print('      and so in order to preserve your $HOME this will now')
 940                 print('      use vendored sources by default.')
 941                 if not os.path.exists(vendor_dir):
 942                     print('error: vendoring required, but vendor directory does not exist.')
 943                     print('       Run `cargo vendor` without sudo to initialize the '
 944                           'vendor directory.')
 945                     raise Exception("{} not found".format(vendor_dir))
 946
 947         if self.use_vendored_sources:
 948             if not os.path.exists('.cargo'):
 949                 os.makedirs('.cargo')
 950             with output('.cargo/config') as cargo_config:
 951                 cargo_config.write(
 952                     "[source.crates-io]\n"
 953                     "replace-with = 'vendored-sources'\n"
 954                     "registry = 'https://example.com'\n"
 955                     "\n"
 956                     "[source.vendored-sources]\n"
 957                     "directory = '{}/vendor'\n"
 958                     .format(self.rust_root))
 959         else:
 960             if os.path.exists('.cargo'):
 961                 shutil.rmtree('.cargo')
 962
 963     def ensure_vendored(self):
 964         """Ensure that the vendored sources are available if needed"""
 965         vendor_dir = os.path.join(self.rust_root, 'vendor')
 966         # Note that this does not handle updating the vendored dependencies if
 967         # the rust git repository is updated. Normal development usually does
 968         # not use vendoring, so hopefully this isn't too much of a problem.
 969         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 970             run([
 971                 self.cargo(),
 972                 "vendor",
 973                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
 974                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
 975             ], verbose=self.verbose, cwd=self.rust_root)
 976
 977
 978 def bootstrap(help_triggered):
 979     """Configure, fetch, build and run the initial bootstrap"""
 980
 981     # If the user is asking for help, let them know that the whole download-and-build
 982     # process has to happen before anything is printed out.
 983     if help_triggered:
 984         print("info: Downloading and building bootstrap before processing --help")
 985         print("      command. See src/bootstrap/README.md for help with common")
 986         print("      commands.")
 987
 988     parser = argparse.ArgumentParser(description='Build rust')
 989     parser.add_argument('--config')
 990     parser.add_argument('--build')
 991     parser.add_argument('--clean', action='store_true')
 992     parser.add_argument('-v', '--verbose', action='count', default=0)
 993
 994     args = [a for a in sys.argv if a != '-h' and a != '--help']
 995     args, _ = parser.parse_known_args(args)
 996
 997     # Configure initial bootstrap
 998     build = RustBuild()
 999     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1000     build.verbose = args.verbose
1001     build.clean = args.clean
1002
1003     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1004     # exists).
1005     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1006     if not toml_path and os.path.exists('config.toml'):
1007         toml_path = 'config.toml'
1008
1009     if toml_path:
1010         if not os.path.exists(toml_path):
1011             toml_path = os.path.join(build.rust_root, toml_path)
1012
1013         with open(toml_path) as config:
1014             build.config_toml = config.read()
1015
1016     profile = build.get_toml('profile')
1017     if profile is not None:
1018         include_file = 'config.{}.toml'.format(profile)
1019         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1020         include_path = os.path.join(include_dir, include_file)
1021         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1022         # specific key, so appending our defaults at the end allows the user to override them
1023         with open(include_path) as included_toml:
1024             build.config_toml += os.linesep + included_toml.read()
1025
1026     config_verbose = build.get_toml('verbose', 'build')
1027     if config_verbose is not None:
1028         build.verbose = max(build.verbose, int(config_verbose))
1029
1030     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1031
1032     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1033
1034     build.check_vendored_status()
1035
1036     build_dir = build.get_toml('build-dir', 'build') or 'build'
1037     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1038
1039     data = stage0_data(build.rust_root)
1040     build.date = data['date']
1041     build.rustc_channel = data['rustc']
1042
1043     if "rustfmt" in data:
1044         build.rustfmt_channel = data['rustfmt']
1045
1046     if 'dev' in data:
1047         build.set_dev_environment()
1048     else:
1049         build.set_normal_environment()
1050
1051     build.update_submodules()
1052
1053     # Fetch/build the bootstrap
1054     build.build = args.build or build.build_triple()
1055     build.download_stage0()
1056     sys.stdout.flush()
1057     build.ensure_vendored()
1058     build.build_bootstrap()
1059     sys.stdout.flush()
1060
1061     # Run the bootstrap
1062     args = [build.bootstrap_binary()]
1063     args.extend(sys.argv[1:])
1064     env = os.environ.copy()
1065     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1066     env["BOOTSTRAP_PYTHON"] = sys.executable
1067     env["BUILD_DIR"] = build.build_dir
1068     env["RUSTC_BOOTSTRAP"] = '1'
1069     if toml_path:
1070         env["BOOTSTRAP_CONFIG"] = toml_path
1071     run(args, env=env, verbose=build.verbose)
1072
1073
1074 def main():
1075     """Entry point for the bootstrap process"""
1076     start_time = time()
1077
1078     # x.py help <cmd> ...
1079     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1080         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1081
1082     help_triggered = (
1083         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1084     try:
1085         bootstrap(help_triggered)
1086         if not help_triggered:
1087             print("Build completed successfully in {}".format(
1088                 format_build_time(time() - start_time)))
1089     except (SystemExit, KeyboardInterrupt) as error:
1090         if hasattr(error, 'code') and isinstance(error.code, int):
1091             exit_code = error.code
1092         else:
1093             exit_code = 1
1094             print(error)
1095         if not help_triggered:
1096             print("Build completed unsuccessfully in {}".format(
1097                 format_build_time(time() - start_time)))
1098         sys.exit(exit_code)
1099
1100
1101 if __name__ == '__main__':
1102     main()