src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple(verbose):
 191     """Build triple as in LLVM"""
 192     # If the user already has a host build triple with an existing `rustc`
 193     # install, use their preference. This fixes most issues with Windows builds
 194     # being detected as GNU instead of MSVC.
 195     default_encoding = sys.getdefaultencoding()
 196     try:
 197         version = subprocess.check_output(["rustc", "--version", "--verbose"])
 198         version = version.decode(default_encoding)
 199         host = next(x for x in version.split('\n') if x.startswith("host: "))
 200         triple = host.split("host: ")[1]
 201         if verbose:
 202             print("detected default triple {}".format(triple))
 203         return triple
 204     except Exception as e:
 205         if verbose:
 206             print("rustup not detected: {}".format(e))
 207             print("falling back to auto-detect")
 208
 209     required = sys.platform != 'win32'
 210     ostype = require(["uname", "-s"], exit=required)
 211     cputype = require(['uname', '-m'], exit=required)
 212
 213     # If we do not have `uname`, assume Windows.
 214     if ostype is None or cputype is None:
 215         return 'x86_64-pc-windows-msvc'
 216
 217     ostype = ostype.decode(default_encoding)
 218     cputype = cputype.decode(default_encoding)
 219
 220     # The goal here is to come up with the same triple as LLVM would,
 221     # at least for the subset of platforms we're willing to target.
 222     ostype_mapper = {
 223         'Darwin': 'apple-darwin',
 224         'DragonFly': 'unknown-dragonfly',
 225         'FreeBSD': 'unknown-freebsd',
 226         'Haiku': 'unknown-haiku',
 227         'NetBSD': 'unknown-netbsd',
 228         'OpenBSD': 'unknown-openbsd'
 229     }
 230
 231     # Consider the direct transformation first and then the special cases
 232     if ostype in ostype_mapper:
 233         ostype = ostype_mapper[ostype]
 234     elif ostype == 'Linux':
 235         os_from_sp = subprocess.check_output(
 236             ['uname', '-o']).strip().decode(default_encoding)
 237         if os_from_sp == 'Android':
 238             ostype = 'linux-android'
 239         else:
 240             ostype = 'unknown-linux-gnu'
 241     elif ostype == 'SunOS':
 242         ostype = 'sun-solaris'
 243         # On Solaris, uname -m will return a machine classification instead
 244         # of a cpu type, so uname -p is recommended instead.  However, the
 245         # output from that option is too generic for our purposes (it will
 246         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 247         # must be used instead.
 248         cputype = require(['isainfo', '-k']).decode(default_encoding)
 249     elif ostype.startswith('MINGW'):
 250         # msys' `uname` does not print gcc configuration, but prints msys
 251         # configuration. so we cannot believe `uname -m`:
 252         # msys1 is always i686 and msys2 is always x86_64.
 253         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 254         # MINGW64 on x86_64.
 255         ostype = 'pc-windows-gnu'
 256         cputype = 'i686'
 257         if os.environ.get('MSYSTEM') == 'MINGW64':
 258             cputype = 'x86_64'
 259     elif ostype.startswith('MSYS'):
 260         ostype = 'pc-windows-gnu'
 261     elif ostype.startswith('CYGWIN_NT'):
 262         cputype = 'i686'
 263         if ostype.endswith('WOW64'):
 264             cputype = 'x86_64'
 265         ostype = 'pc-windows-gnu'
 266     elif sys.platform == 'win32':
 267         # Some Windows platforms might have a `uname` command that returns a
 268         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 269         # these cases, fall back to using sys.platform.
 270         return 'x86_64-pc-windows-msvc'
 271     else:
 272         err = "unknown OS type: {}".format(ostype)
 273         sys.exit(err)
 274
 275     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 276         cputype = subprocess.check_output(
 277               ['uname', '-p']).strip().decode(default_encoding)
 278     cputype_mapper = {
 279         'BePC': 'i686',
 280         'aarch64': 'aarch64',
 281         'amd64': 'x86_64',
 282         'arm64': 'aarch64',
 283         'i386': 'i686',
 284         'i486': 'i686',
 285         'i686': 'i686',
 286         'i786': 'i686',
 287         'powerpc': 'powerpc',
 288         'powerpc64': 'powerpc64',
 289         'powerpc64le': 'powerpc64le',
 290         'ppc': 'powerpc',
 291         'ppc64': 'powerpc64',
 292         'ppc64le': 'powerpc64le',
 293         's390x': 's390x',
 294         'x64': 'x86_64',
 295         'x86': 'i686',
 296         'x86-64': 'x86_64',
 297         'x86_64': 'x86_64'
 298     }
 299
 300     # Consider the direct transformation first and then the special cases
 301     if cputype in cputype_mapper:
 302         cputype = cputype_mapper[cputype]
 303     elif cputype in {'xscale', 'arm'}:
 304         cputype = 'arm'
 305         if ostype == 'linux-android':
 306             ostype = 'linux-androideabi'
 307         elif ostype == 'unknown-freebsd':
 308             cputype = subprocess.check_output(
 309                 ['uname', '-p']).strip().decode(default_encoding)
 310             ostype = 'unknown-freebsd'
 311     elif cputype == 'armv6l':
 312         cputype = 'arm'
 313         if ostype == 'linux-android':
 314             ostype = 'linux-androideabi'
 315         else:
 316             ostype += 'eabihf'
 317     elif cputype in {'armv7l', 'armv8l'}:
 318         cputype = 'armv7'
 319         if ostype == 'linux-android':
 320             ostype = 'linux-androideabi'
 321         else:
 322             ostype += 'eabihf'
 323     elif cputype == 'mips':
 324         if sys.byteorder == 'big':
 325             cputype = 'mips'
 326         elif sys.byteorder == 'little':
 327             cputype = 'mipsel'
 328         else:
 329             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 330     elif cputype == 'mips64':
 331         if sys.byteorder == 'big':
 332             cputype = 'mips64'
 333         elif sys.byteorder == 'little':
 334             cputype = 'mips64el'
 335         else:
 336             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 337         # only the n64 ABI is supported, indicate it
 338         ostype += 'abi64'
 339     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 340         pass
 341     else:
 342         err = "unknown cpu type: {}".format(cputype)
 343         sys.exit(err)
 344
 345     return "{}-{}".format(cputype, ostype)
 346
 347
 348 @contextlib.contextmanager
 349 def output(filepath):
 350     tmp = filepath + '.tmp'
 351     with open(tmp, 'w') as f:
 352         yield f
 353     try:
 354         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 355         os.rename(tmp, filepath)
 356     except OSError:
 357         shutil.copy2(tmp, filepath)
 358         os.remove(tmp)
 359
 360
 361 class RustBuild(object):
 362     """Provide all the methods required to build Rust"""
 363     def __init__(self):
 364         self.date = ''
 365         self._download_url = ''
 366         self.rustc_channel = ''
 367         self.rustfmt_channel = ''
 368         self.build = ''
 369         self.build_dir = ''
 370         self.clean = False
 371         self.config_toml = ''
 372         self.rust_root = ''
 373         self.use_locked_deps = ''
 374         self.use_vendored_sources = ''
 375         self.verbose = False
 376         self.git_version = None
 377         self.nix_deps_dir = None
 378
 379     def download_stage0(self):
 380         """Fetch the build system for Rust, written in Rust
 381
 382         This method will build a cache directory, then it will fetch the
 383         tarball which has the stage0 compiler used to then bootstrap the Rust
 384         compiler itself.
 385
 386         Each downloaded tarball is extracted, after that, the script
 387         will move all the content to the right place.
 388         """
 389         rustc_channel = self.rustc_channel
 390         rustfmt_channel = self.rustfmt_channel
 391
 392         if self.rustc().startswith(self.bin_root()) and \
 393                 (not os.path.exists(self.rustc()) or
 394                  self.program_out_of_date(self.rustc_stamp())):
 395             if os.path.exists(self.bin_root()):
 396                 shutil.rmtree(self.bin_root())
 397             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 398             filename = "rust-std-{}-{}{}".format(
 399                 rustc_channel, self.build, tarball_suffix)
 400             pattern = "rust-std-{}".format(self.build)
 401             self._download_stage0_helper(filename, pattern, tarball_suffix)
 402             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 403                                               tarball_suffix)
 404             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 405             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 406                                               tarball_suffix)
 407             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 408             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 409             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 410             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 411             lib_dir = "{}/lib".format(self.bin_root())
 412             for lib in os.listdir(lib_dir):
 413                 if lib.endswith(".so"):
 414                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 415             with output(self.rustc_stamp()) as rust_stamp:
 416                 rust_stamp.write(self.date)
 417
 418         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 419             not os.path.exists(self.rustfmt())
 420             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 421         ):
 422             if rustfmt_channel:
 423                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 424                 [channel, date] = rustfmt_channel.split('-', 1)
 425                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 426                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 427                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 428                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 429                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 430                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 431
 432         if self.downloading_llvm():
 433             # We want the most recent LLVM submodule update to avoid downloading
 434             # LLVM more often than necessary.
 435             #
 436             # This git command finds that commit SHA, looking for bors-authored
 437             # merges that modified src/llvm-project.
 438             #
 439             # This works even in a repository that has not yet initialized
 440             # submodules.
 441             top_level = subprocess.check_output([
 442                 "git", "rev-parse", "--show-toplevel",
 443             ]).decode(sys.getdefaultencoding()).strip()
 444             llvm_sha = subprocess.check_output([
 445                 "git", "log", "--author=bors", "--format=%H", "-n1",
 446                 "-m", "--first-parent",
 447                 "--",
 448                 "{}/src/llvm-project".format(top_level),
 449                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 450             ]).decode(sys.getdefaultencoding()).strip()
 451             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 452             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 453                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 454                 for binary in ["llvm-config", "FileCheck"]:
 455                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 456                 with output(self.llvm_stamp()) as llvm_stamp:
 457                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 458
 459     def downloading_llvm(self):
 460         opt = self.get_toml('download-ci-llvm', 'llvm')
 461         return opt == "true" \
 462             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 463
 464     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 465         if date is None:
 466             date = self.date
 467         cache_dst = os.path.join(self.build_dir, "cache")
 468         rustc_cache = os.path.join(cache_dst, date)
 469         if not os.path.exists(rustc_cache):
 470             os.makedirs(rustc_cache)
 471
 472         url = "{}/dist/{}".format(self._download_url, date)
 473         tarball = os.path.join(rustc_cache, filename)
 474         if not os.path.exists(tarball):
 475             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 476         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 477
 478     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 479         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 480         cache_dst = os.path.join(self.build_dir, "cache")
 481         rustc_cache = os.path.join(cache_dst, cache_prefix)
 482         if not os.path.exists(rustc_cache):
 483             os.makedirs(rustc_cache)
 484
 485         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 486         if llvm_assertions:
 487             url = url.replace('rustc-builds', 'rustc-builds-alt')
 488         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 489         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 490         tarball = os.path.join(rustc_cache, filename)
 491         if not os.path.exists(tarball):
 492             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 493         unpack(tarball, tarball_suffix, self.llvm_root(),
 494                 match="rust-dev",
 495                 verbose=self.verbose)
 496
 497     def fix_bin_or_dylib(self, fname):
 498         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 499         or the RPATH section, to fix the dynamic library search path
 500
 501         This method is only required on NixOS and uses the PatchELF utility to
 502         change the interpreter/RPATH of ELF executables.
 503
 504         Please see https://nixos.org/patchelf.html for more information
 505         """
 506         default_encoding = sys.getdefaultencoding()
 507         try:
 508             ostype = subprocess.check_output(
 509                 ['uname', '-s']).strip().decode(default_encoding)
 510         except subprocess.CalledProcessError:
 511             return
 512         except OSError as reason:
 513             if getattr(reason, 'winerror', None) is not None:
 514                 return
 515             raise reason
 516
 517         if ostype != "Linux":
 518             return
 519
 520         if not os.path.exists("/etc/NIXOS"):
 521             return
 522         if os.path.exists("/lib"):
 523             return
 524
 525         # At this point we're pretty sure the user is running NixOS
 526         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 527         print(nix_os_msg, fname)
 528
 529         # Only build `stage0/.nix-deps` once.
 530         nix_deps_dir = self.nix_deps_dir
 531         if not nix_deps_dir:
 532             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 533             if not os.path.exists(nix_deps_dir):
 534                 os.makedirs(nix_deps_dir)
 535
 536             nix_deps = [
 537                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 538                 "stdenv.cc.bintools",
 539
 540                 # Needed as a system dependency of `libLLVM-*.so`.
 541                 "zlib",
 542
 543                 # Needed for patching ELF binaries (see doc comment above).
 544                 "patchelf",
 545             ]
 546
 547             # Run `nix-build` to "build" each dependency (which will likely reuse
 548             # the existing `/nix/store` copy, or at most download a pre-built copy).
 549             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 550             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 551             # ensuring garbage collection will never remove the `/nix/store` path
 552             # (which would break our patched binaries that hardcode those paths).
 553             for dep in nix_deps:
 554                 try:
 555                     subprocess.check_output([
 556                         "nix-build", "<nixpkgs>",
 557                         "-A", dep,
 558                         "-o", "{}/{}".format(nix_deps_dir, dep),
 559                     ])
 560                 except subprocess.CalledProcessError as reason:
 561                     print("warning: failed to call nix-build:", reason)
 562                     return
 563
 564             self.nix_deps_dir = nix_deps_dir
 565
 566         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 567
 568         if fname.endswith(".so"):
 569             # Dynamic library, patch RPATH to point to system dependencies.
 570             dylib_deps = ["zlib"]
 571             rpath_entries = [
 572                 # Relative default, all binary and dynamic libraries we ship
 573                 # appear to have this (even when `../lib` is redundant).
 574                 "$ORIGIN/../lib",
 575             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 576             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 577         else:
 578             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 579             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 580                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 581
 582         try:
 583             subprocess.check_output([patchelf] + patchelf_args + [fname])
 584         except subprocess.CalledProcessError as reason:
 585             print("warning: failed to call patchelf:", reason)
 586             return
 587
 588     def rustc_stamp(self):
 589         """Return the path for .rustc-stamp
 590
 591         >>> rb = RustBuild()
 592         >>> rb.build_dir = "build"
 593         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 594         True
 595         """
 596         return os.path.join(self.bin_root(), '.rustc-stamp')
 597
 598     def rustfmt_stamp(self):
 599         """Return the path for .rustfmt-stamp
 600
 601         >>> rb = RustBuild()
 602         >>> rb.build_dir = "build"
 603         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 604         True
 605         """
 606         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 607
 608     def llvm_stamp(self):
 609         """Return the path for .rustfmt-stamp
 610
 611         >>> rb = RustBuild()
 612         >>> rb.build_dir = "build"
 613         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 614         True
 615         """
 616         return os.path.join(self.llvm_root(), '.llvm-stamp')
 617
 618
 619     def program_out_of_date(self, stamp_path, extra=""):
 620         """Check if the given program stamp is out of date"""
 621         if not os.path.exists(stamp_path) or self.clean:
 622             return True
 623         with open(stamp_path, 'r') as stamp:
 624             return (self.date + extra) != stamp.read()
 625
 626     def bin_root(self):
 627         """Return the binary root directory
 628
 629         >>> rb = RustBuild()
 630         >>> rb.build_dir = "build"
 631         >>> rb.bin_root() == os.path.join("build", "stage0")
 632         True
 633
 634         When the 'build' property is given should be a nested directory:
 635
 636         >>> rb.build = "devel"
 637         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 638         True
 639         """
 640         return os.path.join(self.build_dir, self.build, "stage0")
 641
 642     def llvm_root(self):
 643         """Return the CI LLVM root directory
 644
 645         >>> rb = RustBuild()
 646         >>> rb.build_dir = "build"
 647         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 648         True
 649
 650         When the 'build' property is given should be a nested directory:
 651
 652         >>> rb.build = "devel"
 653         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 654         True
 655         """
 656         return os.path.join(self.build_dir, self.build, "ci-llvm")
 657
 658     def get_toml(self, key, section=None):
 659         """Returns the value of the given key in config.toml, otherwise returns None
 660
 661         >>> rb = RustBuild()
 662         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 663         >>> rb.get_toml("key2")
 664         'value2'
 665
 666         If the key does not exists, the result is None:
 667
 668         >>> rb.get_toml("key3") is None
 669         True
 670
 671         Optionally also matches the section the key appears in
 672
 673         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 674         >>> rb.get_toml('key', 'a')
 675         'value1'
 676         >>> rb.get_toml('key', 'b')
 677         'value2'
 678         >>> rb.get_toml('key', 'c') is None
 679         True
 680
 681         >>> rb.config_toml = 'key1 = true'
 682         >>> rb.get_toml("key1")
 683         'true'
 684         """
 685
 686         cur_section = None
 687         for line in self.config_toml.splitlines():
 688             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 689             if section_match is not None:
 690                 cur_section = section_match.group(1)
 691
 692             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 693             if match is not None:
 694                 value = match.group(1)
 695                 if section is None or section == cur_section:
 696                     return self.get_string(value) or value.strip()
 697         return None
 698
 699     def cargo(self):
 700         """Return config path for cargo"""
 701         return self.program_config('cargo')
 702
 703     def rustc(self):
 704         """Return config path for rustc"""
 705         return self.program_config('rustc')
 706
 707     def rustfmt(self):
 708         """Return config path for rustfmt"""
 709         if not self.rustfmt_channel:
 710             return None
 711         return self.program_config('rustfmt')
 712
 713     def program_config(self, program):
 714         """Return config path for the given program
 715
 716         >>> rb = RustBuild()
 717         >>> rb.config_toml = 'rustc = "rustc"\\n'
 718         >>> rb.program_config('rustc')
 719         'rustc'
 720         >>> rb.config_toml = ''
 721         >>> cargo_path = rb.program_config('cargo')
 722         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 723         ... "bin", "cargo")
 724         True
 725         """
 726         config = self.get_toml(program)
 727         if config:
 728             return os.path.expanduser(config)
 729         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 730             program, self.exe_suffix()))
 731
 732     @staticmethod
 733     def get_string(line):
 734         """Return the value between double quotes
 735
 736         >>> RustBuild.get_string('    "devel"   ')
 737         'devel'
 738         >>> RustBuild.get_string("    'devel'   ")
 739         'devel'
 740         >>> RustBuild.get_string('devel') is None
 741         True
 742         >>> RustBuild.get_string('    "devel   ')
 743         ''
 744         """
 745         start = line.find('"')
 746         if start != -1:
 747             end = start + 1 + line[start + 1:].find('"')
 748             return line[start + 1:end]
 749         start = line.find('\'')
 750         if start != -1:
 751             end = start + 1 + line[start + 1:].find('\'')
 752             return line[start + 1:end]
 753         return None
 754
 755     @staticmethod
 756     def exe_suffix():
 757         """Return a suffix for executables"""
 758         if sys.platform == 'win32':
 759             return '.exe'
 760         return ''
 761
 762     def bootstrap_binary(self):
 763         """Return the path of the bootstrap binary
 764
 765         >>> rb = RustBuild()
 766         >>> rb.build_dir = "build"
 767         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 768         ... "debug", "bootstrap")
 769         True
 770         """
 771         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 772
 773     def build_bootstrap(self):
 774         """Build bootstrap"""
 775         build_dir = os.path.join(self.build_dir, "bootstrap")
 776         if self.clean and os.path.exists(build_dir):
 777             shutil.rmtree(build_dir)
 778         env = os.environ.copy()
 779         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 780         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 781         if "CARGO_BUILD_TARGET" in env:
 782             del env["CARGO_BUILD_TARGET"]
 783         env["CARGO_TARGET_DIR"] = build_dir
 784         env["RUSTC"] = self.rustc()
 785         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 786             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 787             if "LD_LIBRARY_PATH" in env else ""
 788         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 789             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 790             if "DYLD_LIBRARY_PATH" in env else ""
 791         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 792             (os.pathsep + env["LIBRARY_PATH"]) \
 793             if "LIBRARY_PATH" in env else ""
 794         # preserve existing RUSTFLAGS
 795         env.setdefault("RUSTFLAGS", "")
 796         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 797
 798         build_section = "target.{}".format(self.build)
 799         target_features = []
 800         if self.get_toml("crt-static", build_section) == "true":
 801             target_features += ["+crt-static"]
 802         elif self.get_toml("crt-static", build_section) == "false":
 803             target_features += ["-crt-static"]
 804         if target_features:
 805             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 806         target_linker = self.get_toml("linker", build_section)
 807         if target_linker is not None:
 808             env["RUSTFLAGS"] += " -C linker=" + target_linker
 809         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 810         if self.get_toml("deny-warnings", "rust") != "false":
 811             env["RUSTFLAGS"] += " -Dwarnings"
 812
 813         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 814             os.pathsep + env["PATH"]
 815         if not os.path.isfile(self.cargo()):
 816             raise Exception("no cargo executable found at `{}`".format(
 817                 self.cargo()))
 818         args = [self.cargo(), "build", "--manifest-path",
 819                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 820         for _ in range(1, self.verbose):
 821             args.append("--verbose")
 822         if self.use_locked_deps:
 823             args.append("--locked")
 824         if self.use_vendored_sources:
 825             args.append("--frozen")
 826         run(args, env=env, verbose=self.verbose)
 827
 828     def build_triple(self):
 829         """Build triple as in LLVM
 830
 831         Note that `default_build_triple` is moderately expensive,
 832         so use `self.build` where possible.
 833         """
 834         config = self.get_toml('build')
 835         if config:
 836             return config
 837         return default_build_triple(self.verbose)
 838
 839     def check_submodule(self, module, slow_submodules):
 840         if not slow_submodules:
 841             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 842                                            cwd=os.path.join(self.rust_root, module),
 843                                            stdout=subprocess.PIPE)
 844             return checked_out
 845         else:
 846             return None
 847
 848     def update_submodule(self, module, checked_out, recorded_submodules):
 849         module_path = os.path.join(self.rust_root, module)
 850
 851         if checked_out is not None:
 852             default_encoding = sys.getdefaultencoding()
 853             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 854             if recorded_submodules[module] == checked_out:
 855                 return
 856
 857         print("Updating submodule", module)
 858
 859         run(["git", "submodule", "-q", "sync", module],
 860             cwd=self.rust_root, verbose=self.verbose)
 861
 862         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 863         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 864             update_args.append("--progress")
 865         update_args.append(module)
 866         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 867
 868         run(["git", "reset", "-q", "--hard"],
 869             cwd=module_path, verbose=self.verbose)
 870         run(["git", "clean", "-qdfx"],
 871             cwd=module_path, verbose=self.verbose)
 872
 873     def update_submodules(self):
 874         """Update submodules"""
 875         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 876                 self.get_toml('submodules') == "false":
 877             return
 878
 879         default_encoding = sys.getdefaultencoding()
 880
 881         # check the existence and version of 'git' command
 882         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 883         self.git_version = distutils.version.LooseVersion(git_version_str)
 884
 885         slow_submodules = self.get_toml('fast-submodules') == "false"
 886         start_time = time()
 887         if slow_submodules:
 888             print('Unconditionally updating all submodules')
 889         else:
 890             print('Updating only changed submodules')
 891         default_encoding = sys.getdefaultencoding()
 892         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 893             ["git", "config", "--file",
 894              os.path.join(self.rust_root, ".gitmodules"),
 895              "--get-regexp", "path"]
 896         ).decode(default_encoding).splitlines()]
 897         filtered_submodules = []
 898         submodules_names = []
 899         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
 900         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
 901         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
 902             or "llvm" in self.get_toml('codegen-backends', 'rust')
 903         for module in submodules:
 904             if module.endswith("llvm-project"):
 905                 # Don't sync the llvm-project submodule if an external LLVM was
 906                 # provided, if we are downloading LLVM or if the LLVM backend is
 907                 # not being built. Also, if the submodule has been initialized
 908                 # already, sync it anyways so that it doesn't mess up contributor
 909                 # pull requests.
 910                 if external_llvm_provided or not llvm_needed:
 911                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
 912                         continue
 913             check = self.check_submodule(module, slow_submodules)
 914             filtered_submodules.append((module, check))
 915             submodules_names.append(module)
 916         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 917                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 918         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 919         recorded_submodules = {}
 920         for data in recorded:
 921             data = data.split()
 922             recorded_submodules[data[3]] = data[2]
 923         for module in filtered_submodules:
 924             self.update_submodule(module[0], module[1], recorded_submodules)
 925         print("Submodules updated in %.2f seconds" % (time() - start_time))
 926
 927     def set_normal_environment(self):
 928         """Set download URL for normal environment"""
 929         if 'RUSTUP_DIST_SERVER' in os.environ:
 930             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 931         else:
 932             self._download_url = 'https://static.rust-lang.org'
 933
 934     def set_dev_environment(self):
 935         """Set download URL for development environment"""
 936         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 937             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 938         else:
 939             self._download_url = 'https://dev-static.rust-lang.org'
 940
 941     def check_vendored_status(self):
 942         """Check that vendoring is configured properly"""
 943         vendor_dir = os.path.join(self.rust_root, 'vendor')
 944         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 945             if os.environ.get('USER') != os.environ['SUDO_USER']:
 946                 self.use_vendored_sources = True
 947                 print('info: looks like you are running this command under `sudo`')
 948                 print('      and so in order to preserve your $HOME this will now')
 949                 print('      use vendored sources by default.')
 950                 if not os.path.exists(vendor_dir):
 951                     print('error: vendoring required, but vendor directory does not exist.')
 952                     print('       Run `cargo vendor` without sudo to initialize the '
 953                           'vendor directory.')
 954                     raise Exception("{} not found".format(vendor_dir))
 955
 956         if self.use_vendored_sources:
 957             if not os.path.exists('.cargo'):
 958                 os.makedirs('.cargo')
 959             with output('.cargo/config') as cargo_config:
 960                 cargo_config.write(
 961                     "[source.crates-io]\n"
 962                     "replace-with = 'vendored-sources'\n"
 963                     "registry = 'https://example.com'\n"
 964                     "\n"
 965                     "[source.vendored-sources]\n"
 966                     "directory = '{}/vendor'\n"
 967                     .format(self.rust_root))
 968         else:
 969             if os.path.exists('.cargo'):
 970                 shutil.rmtree('.cargo')
 971
 972     def ensure_vendored(self):
 973         """Ensure that the vendored sources are available if needed"""
 974         vendor_dir = os.path.join(self.rust_root, 'vendor')
 975         # Note that this does not handle updating the vendored dependencies if
 976         # the rust git repository is updated. Normal development usually does
 977         # not use vendoring, so hopefully this isn't too much of a problem.
 978         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 979             run([
 980                 self.cargo(),
 981                 "vendor",
 982                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
 983                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
 984             ], verbose=self.verbose, cwd=self.rust_root)
 985
 986
 987 def bootstrap(help_triggered):
 988     """Configure, fetch, build and run the initial bootstrap"""
 989
 990     # If the user is asking for help, let them know that the whole download-and-build
 991     # process has to happen before anything is printed out.
 992     if help_triggered:
 993         print("info: Downloading and building bootstrap before processing --help")
 994         print("      command. See src/bootstrap/README.md for help with common")
 995         print("      commands.")
 996
 997     parser = argparse.ArgumentParser(description='Build rust')
 998     parser.add_argument('--config')
 999     parser.add_argument('--build')
1000     parser.add_argument('--clean', action='store_true')
1001     parser.add_argument('-v', '--verbose', action='count', default=0)
1002
1003     args = [a for a in sys.argv if a != '-h' and a != '--help']
1004     args, _ = parser.parse_known_args(args)
1005
1006     # Configure initial bootstrap
1007     build = RustBuild()
1008     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1009     build.verbose = args.verbose
1010     build.clean = args.clean
1011
1012     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1013     # exists).
1014     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1015     if not toml_path and os.path.exists('config.toml'):
1016         toml_path = 'config.toml'
1017
1018     if toml_path:
1019         if not os.path.exists(toml_path):
1020             toml_path = os.path.join(build.rust_root, toml_path)
1021
1022         with open(toml_path) as config:
1023             build.config_toml = config.read()
1024
1025     profile = build.get_toml('profile')
1026     if profile is not None:
1027         include_file = 'config.{}.toml'.format(profile)
1028         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1029         include_path = os.path.join(include_dir, include_file)
1030         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1031         # specific key, so appending our defaults at the end allows the user to override them
1032         with open(include_path) as included_toml:
1033             build.config_toml += os.linesep + included_toml.read()
1034
1035     config_verbose = build.get_toml('verbose', 'build')
1036     if config_verbose is not None:
1037         build.verbose = max(build.verbose, int(config_verbose))
1038
1039     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1040
1041     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1042
1043     build.check_vendored_status()
1044
1045     build_dir = build.get_toml('build-dir', 'build') or 'build'
1046     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1047
1048     data = stage0_data(build.rust_root)
1049     build.date = data['date']
1050     build.rustc_channel = data['rustc']
1051
1052     if "rustfmt" in data:
1053         build.rustfmt_channel = data['rustfmt']
1054
1055     if 'dev' in data:
1056         build.set_dev_environment()
1057     else:
1058         build.set_normal_environment()
1059
1060     build.update_submodules()
1061
1062     # Fetch/build the bootstrap
1063     build.build = args.build or build.build_triple()
1064     build.download_stage0()
1065     sys.stdout.flush()
1066     build.ensure_vendored()
1067     build.build_bootstrap()
1068     sys.stdout.flush()
1069
1070     # Run the bootstrap
1071     args = [build.bootstrap_binary()]
1072     args.extend(sys.argv[1:])
1073     env = os.environ.copy()
1074     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1075     env["BOOTSTRAP_PYTHON"] = sys.executable
1076     env["BUILD_DIR"] = build.build_dir
1077     env["RUSTC_BOOTSTRAP"] = '1'
1078     if toml_path:
1079         env["BOOTSTRAP_CONFIG"] = toml_path
1080     run(args, env=env, verbose=build.verbose)
1081
1082
1083 def main():
1084     """Entry point for the bootstrap process"""
1085     start_time = time()
1086
1087     # x.py help <cmd> ...
1088     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1089         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1090
1091     help_triggered = (
1092         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1093     try:
1094         bootstrap(help_triggered)
1095         if not help_triggered:
1096             print("Build completed successfully in {}".format(
1097                 format_build_time(time() - start_time)))
1098     except (SystemExit, KeyboardInterrupt) as error:
1099         if hasattr(error, 'code') and isinstance(error.code, int):
1100             exit_code = error.code
1101         else:
1102             exit_code = 1
1103             print(error)
1104         if not help_triggered:
1105             print("Build completed unsuccessfully in {}".format(
1106                 format_build_time(time() - start_time)))
1107         sys.exit(exit_code)
1108
1109
1110 if __name__ == '__main__':
1111     main()