src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple(verbose):
 191     """Build triple as in LLVM"""
 192     # If the user already has a host build triple with an existing `rustc`
 193     # install, use their preference. This fixes most issues with Windows builds
 194     # being detected as GNU instead of MSVC.
 195     default_encoding = sys.getdefaultencoding()
 196     try:
 197         version = subprocess.check_output(["rustc", "--version", "--verbose"])
 198         version = version.decode(default_encoding)
 199         host = next(x for x in version.split('\n') if x.startswith("host: "))
 200         triple = host.split("host: ")[1]
 201         if verbose:
 202             print("detected default triple {}".format(triple))
 203         return triple
 204     except Exception as e:
 205         if verbose:
 206             print("rustup not detected: {}".format(e))
 207             print("falling back to auto-detect")
 208
 209     required = sys.platform != 'win32'
 210     ostype = require(["uname", "-s"], exit=required)
 211     cputype = require(['uname', '-m'], exit=required)
 212
 213     # If we do not have `uname`, assume Windows.
 214     if ostype is None or cputype is None:
 215         return 'x86_64-pc-windows-msvc'
 216
 217     ostype = ostype.decode(default_encoding)
 218     cputype = cputype.decode(default_encoding)
 219
 220     # The goal here is to come up with the same triple as LLVM would,
 221     # at least for the subset of platforms we're willing to target.
 222     ostype_mapper = {
 223         'Darwin': 'apple-darwin',
 224         'DragonFly': 'unknown-dragonfly',
 225         'FreeBSD': 'unknown-freebsd',
 226         'Haiku': 'unknown-haiku',
 227         'NetBSD': 'unknown-netbsd',
 228         'OpenBSD': 'unknown-openbsd'
 229     }
 230
 231     # Consider the direct transformation first and then the special cases
 232     if ostype in ostype_mapper:
 233         ostype = ostype_mapper[ostype]
 234     elif ostype == 'Linux':
 235         os_from_sp = subprocess.check_output(
 236             ['uname', '-o']).strip().decode(default_encoding)
 237         if os_from_sp == 'Android':
 238             ostype = 'linux-android'
 239         else:
 240             ostype = 'unknown-linux-gnu'
 241     elif ostype == 'SunOS':
 242         ostype = 'sun-solaris'
 243         # On Solaris, uname -m will return a machine classification instead
 244         # of a cpu type, so uname -p is recommended instead.  However, the
 245         # output from that option is too generic for our purposes (it will
 246         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 247         # must be used instead.
 248         cputype = require(['isainfo', '-k']).decode(default_encoding)
 249     elif ostype.startswith('MINGW'):
 250         # msys' `uname` does not print gcc configuration, but prints msys
 251         # configuration. so we cannot believe `uname -m`:
 252         # msys1 is always i686 and msys2 is always x86_64.
 253         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 254         # MINGW64 on x86_64.
 255         ostype = 'pc-windows-gnu'
 256         cputype = 'i686'
 257         if os.environ.get('MSYSTEM') == 'MINGW64':
 258             cputype = 'x86_64'
 259     elif ostype.startswith('MSYS'):
 260         ostype = 'pc-windows-gnu'
 261     elif ostype.startswith('CYGWIN_NT'):
 262         cputype = 'i686'
 263         if ostype.endswith('WOW64'):
 264             cputype = 'x86_64'
 265         ostype = 'pc-windows-gnu'
 266     elif sys.platform == 'win32':
 267         # Some Windows platforms might have a `uname` command that returns a
 268         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 269         # these cases, fall back to using sys.platform.
 270         return 'x86_64-pc-windows-msvc'
 271     else:
 272         err = "unknown OS type: {}".format(ostype)
 273         sys.exit(err)
 274
 275     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 276         cputype = subprocess.check_output(
 277               ['uname', '-p']).strip().decode(default_encoding)
 278     cputype_mapper = {
 279         'BePC': 'i686',
 280         'aarch64': 'aarch64',
 281         'amd64': 'x86_64',
 282         'arm64': 'aarch64',
 283         'i386': 'i686',
 284         'i486': 'i686',
 285         'i686': 'i686',
 286         'i786': 'i686',
 287         'powerpc': 'powerpc',
 288         'powerpc64': 'powerpc64',
 289         'powerpc64le': 'powerpc64le',
 290         'ppc': 'powerpc',
 291         'ppc64': 'powerpc64',
 292         'ppc64le': 'powerpc64le',
 293         's390x': 's390x',
 294         'x64': 'x86_64',
 295         'x86': 'i686',
 296         'x86-64': 'x86_64',
 297         'x86_64': 'x86_64'
 298     }
 299
 300     # Consider the direct transformation first and then the special cases
 301     if cputype in cputype_mapper:
 302         cputype = cputype_mapper[cputype]
 303     elif cputype in {'xscale', 'arm'}:
 304         cputype = 'arm'
 305         if ostype == 'linux-android':
 306             ostype = 'linux-androideabi'
 307         elif ostype == 'unknown-freebsd':
 308             cputype = subprocess.check_output(
 309                 ['uname', '-p']).strip().decode(default_encoding)
 310             ostype = 'unknown-freebsd'
 311     elif cputype == 'armv6l':
 312         cputype = 'arm'
 313         if ostype == 'linux-android':
 314             ostype = 'linux-androideabi'
 315         else:
 316             ostype += 'eabihf'
 317     elif cputype in {'armv7l', 'armv8l'}:
 318         cputype = 'armv7'
 319         if ostype == 'linux-android':
 320             ostype = 'linux-androideabi'
 321         else:
 322             ostype += 'eabihf'
 323     elif cputype == 'mips':
 324         if sys.byteorder == 'big':
 325             cputype = 'mips'
 326         elif sys.byteorder == 'little':
 327             cputype = 'mipsel'
 328         else:
 329             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 330     elif cputype == 'mips64':
 331         if sys.byteorder == 'big':
 332             cputype = 'mips64'
 333         elif sys.byteorder == 'little':
 334             cputype = 'mips64el'
 335         else:
 336             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 337         # only the n64 ABI is supported, indicate it
 338         ostype += 'abi64'
 339     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 340         pass
 341     else:
 342         err = "unknown cpu type: {}".format(cputype)
 343         sys.exit(err)
 344
 345     return "{}-{}".format(cputype, ostype)
 346
 347
 348 @contextlib.contextmanager
 349 def output(filepath):
 350     tmp = filepath + '.tmp'
 351     with open(tmp, 'w') as f:
 352         yield f
 353     try:
 354         if os.path.exists(filepath):
 355             os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 356     except OSError:
 357         shutil.copy2(tmp, filepath)
 358         os.remove(tmp)
 359         return
 360     os.rename(tmp, filepath)
 361
 362
 363 class RustBuild(object):
 364     """Provide all the methods required to build Rust"""
 365     def __init__(self):
 366         self.date = ''
 367         self._download_url = ''
 368         self.rustc_channel = ''
 369         self.rustfmt_channel = ''
 370         self.build = ''
 371         self.build_dir = ''
 372         self.clean = False
 373         self.config_toml = ''
 374         self.rust_root = ''
 375         self.use_locked_deps = ''
 376         self.use_vendored_sources = ''
 377         self.verbose = False
 378         self.git_version = None
 379         self.nix_deps_dir = None
 380
 381     def download_stage0(self):
 382         """Fetch the build system for Rust, written in Rust
 383
 384         This method will build a cache directory, then it will fetch the
 385         tarball which has the stage0 compiler used to then bootstrap the Rust
 386         compiler itself.
 387
 388         Each downloaded tarball is extracted, after that, the script
 389         will move all the content to the right place.
 390         """
 391         rustc_channel = self.rustc_channel
 392         rustfmt_channel = self.rustfmt_channel
 393
 394         if self.rustc().startswith(self.bin_root()) and \
 395                 (not os.path.exists(self.rustc()) or
 396                  self.program_out_of_date(self.rustc_stamp())):
 397             if os.path.exists(self.bin_root()):
 398                 shutil.rmtree(self.bin_root())
 399             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 400             filename = "rust-std-{}-{}{}".format(
 401                 rustc_channel, self.build, tarball_suffix)
 402             pattern = "rust-std-{}".format(self.build)
 403             self._download_stage0_helper(filename, pattern, tarball_suffix)
 404             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 405                                               tarball_suffix)
 406             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 407             filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
 408                                               tarball_suffix)
 409             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 410             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 411             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 412             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 413             lib_dir = "{}/lib".format(self.bin_root())
 414             for lib in os.listdir(lib_dir):
 415                 if lib.endswith(".so"):
 416                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 417             with output(self.rustc_stamp()) as rust_stamp:
 418                 rust_stamp.write(self.date)
 419
 420         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 421             not os.path.exists(self.rustfmt())
 422             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 423         ):
 424             if rustfmt_channel:
 425                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 426                 [channel, date] = rustfmt_channel.split('-', 1)
 427                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 428                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 429                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 430                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 431                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 432                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 433
 434         if self.downloading_llvm():
 435             # We want the most recent LLVM submodule update to avoid downloading
 436             # LLVM more often than necessary.
 437             #
 438             # This git command finds that commit SHA, looking for bors-authored
 439             # merges that modified src/llvm-project.
 440             #
 441             # This works even in a repository that has not yet initialized
 442             # submodules.
 443             top_level = subprocess.check_output([
 444                 "git", "rev-parse", "--show-toplevel",
 445             ]).decode(sys.getdefaultencoding()).strip()
 446             llvm_sha = subprocess.check_output([
 447                 "git", "log", "--author=bors", "--format=%H", "-n1",
 448                 "-m", "--first-parent",
 449                 "--",
 450                 "{}/src/llvm-project".format(top_level),
 451                 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
 452             ]).decode(sys.getdefaultencoding()).strip()
 453             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 454             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 455                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 456                 for binary in ["llvm-config", "FileCheck"]:
 457                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 458                 with output(self.llvm_stamp()) as llvm_stamp:
 459                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 460
 461     def downloading_llvm(self):
 462         opt = self.get_toml('download-ci-llvm', 'llvm')
 463         return opt == "true" \
 464             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 465
 466     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 467         if date is None:
 468             date = self.date
 469         cache_dst = os.path.join(self.build_dir, "cache")
 470         rustc_cache = os.path.join(cache_dst, date)
 471         if not os.path.exists(rustc_cache):
 472             os.makedirs(rustc_cache)
 473
 474         url = "{}/dist/{}".format(self._download_url, date)
 475         tarball = os.path.join(rustc_cache, filename)
 476         if not os.path.exists(tarball):
 477             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 478         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 479
 480     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 481         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 482         cache_dst = os.path.join(self.build_dir, "cache")
 483         rustc_cache = os.path.join(cache_dst, cache_prefix)
 484         if not os.path.exists(rustc_cache):
 485             os.makedirs(rustc_cache)
 486
 487         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 488         if llvm_assertions:
 489             url = url.replace('rustc-builds', 'rustc-builds-alt')
 490         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 491         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 492         tarball = os.path.join(rustc_cache, filename)
 493         if not os.path.exists(tarball):
 494             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 495         unpack(tarball, tarball_suffix, self.llvm_root(),
 496                 match="rust-dev",
 497                 verbose=self.verbose)
 498
 499     def fix_bin_or_dylib(self, fname):
 500         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 501         or the RPATH section, to fix the dynamic library search path
 502
 503         This method is only required on NixOS and uses the PatchELF utility to
 504         change the interpreter/RPATH of ELF executables.
 505
 506         Please see https://nixos.org/patchelf.html for more information
 507         """
 508         default_encoding = sys.getdefaultencoding()
 509         try:
 510             ostype = subprocess.check_output(
 511                 ['uname', '-s']).strip().decode(default_encoding)
 512         except subprocess.CalledProcessError:
 513             return
 514         except OSError as reason:
 515             if getattr(reason, 'winerror', None) is not None:
 516                 return
 517             raise reason
 518
 519         if ostype != "Linux":
 520             return
 521
 522         if not os.path.exists("/etc/NIXOS"):
 523             return
 524         if os.path.exists("/lib"):
 525             return
 526
 527         # At this point we're pretty sure the user is running NixOS
 528         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 529         print(nix_os_msg, fname)
 530
 531         # Only build `stage0/.nix-deps` once.
 532         nix_deps_dir = self.nix_deps_dir
 533         if not nix_deps_dir:
 534             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 535             if not os.path.exists(nix_deps_dir):
 536                 os.makedirs(nix_deps_dir)
 537
 538             nix_deps = [
 539                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 540                 "stdenv.cc.bintools",
 541
 542                 # Needed as a system dependency of `libLLVM-*.so`.
 543                 "zlib",
 544
 545                 # Needed for patching ELF binaries (see doc comment above).
 546                 "patchelf",
 547             ]
 548
 549             # Run `nix-build` to "build" each dependency (which will likely reuse
 550             # the existing `/nix/store` copy, or at most download a pre-built copy).
 551             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 552             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 553             # ensuring garbage collection will never remove the `/nix/store` path
 554             # (which would break our patched binaries that hardcode those paths).
 555             for dep in nix_deps:
 556                 try:
 557                     subprocess.check_output([
 558                         "nix-build", "<nixpkgs>",
 559                         "-A", dep,
 560                         "-o", "{}/{}".format(nix_deps_dir, dep),
 561                     ])
 562                 except subprocess.CalledProcessError as reason:
 563                     print("warning: failed to call nix-build:", reason)
 564                     return
 565
 566             self.nix_deps_dir = nix_deps_dir
 567
 568         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 569
 570         if fname.endswith(".so"):
 571             # Dynamic library, patch RPATH to point to system dependencies.
 572             dylib_deps = ["zlib"]
 573             rpath_entries = [
 574                 # Relative default, all binary and dynamic libraries we ship
 575                 # appear to have this (even when `../lib` is redundant).
 576                 "$ORIGIN/../lib",
 577             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 578             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 579         else:
 580             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 581             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 582                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 583
 584         try:
 585             subprocess.check_output([patchelf] + patchelf_args + [fname])
 586         except subprocess.CalledProcessError as reason:
 587             print("warning: failed to call patchelf:", reason)
 588             return
 589
 590     def rustc_stamp(self):
 591         """Return the path for .rustc-stamp
 592
 593         >>> rb = RustBuild()
 594         >>> rb.build_dir = "build"
 595         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 596         True
 597         """
 598         return os.path.join(self.bin_root(), '.rustc-stamp')
 599
 600     def rustfmt_stamp(self):
 601         """Return the path for .rustfmt-stamp
 602
 603         >>> rb = RustBuild()
 604         >>> rb.build_dir = "build"
 605         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 606         True
 607         """
 608         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 609
 610     def llvm_stamp(self):
 611         """Return the path for .rustfmt-stamp
 612
 613         >>> rb = RustBuild()
 614         >>> rb.build_dir = "build"
 615         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 616         True
 617         """
 618         return os.path.join(self.llvm_root(), '.llvm-stamp')
 619
 620
 621     def program_out_of_date(self, stamp_path, extra=""):
 622         """Check if the given program stamp is out of date"""
 623         if not os.path.exists(stamp_path) or self.clean:
 624             return True
 625         with open(stamp_path, 'r') as stamp:
 626             return (self.date + extra) != stamp.read()
 627
 628     def bin_root(self):
 629         """Return the binary root directory
 630
 631         >>> rb = RustBuild()
 632         >>> rb.build_dir = "build"
 633         >>> rb.bin_root() == os.path.join("build", "stage0")
 634         True
 635
 636         When the 'build' property is given should be a nested directory:
 637
 638         >>> rb.build = "devel"
 639         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 640         True
 641         """
 642         return os.path.join(self.build_dir, self.build, "stage0")
 643
 644     def llvm_root(self):
 645         """Return the CI LLVM root directory
 646
 647         >>> rb = RustBuild()
 648         >>> rb.build_dir = "build"
 649         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 650         True
 651
 652         When the 'build' property is given should be a nested directory:
 653
 654         >>> rb.build = "devel"
 655         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 656         True
 657         """
 658         return os.path.join(self.build_dir, self.build, "ci-llvm")
 659
 660     def get_toml(self, key, section=None):
 661         """Returns the value of the given key in config.toml, otherwise returns None
 662
 663         >>> rb = RustBuild()
 664         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 665         >>> rb.get_toml("key2")
 666         'value2'
 667
 668         If the key does not exists, the result is None:
 669
 670         >>> rb.get_toml("key3") is None
 671         True
 672
 673         Optionally also matches the section the key appears in
 674
 675         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 676         >>> rb.get_toml('key', 'a')
 677         'value1'
 678         >>> rb.get_toml('key', 'b')
 679         'value2'
 680         >>> rb.get_toml('key', 'c') is None
 681         True
 682
 683         >>> rb.config_toml = 'key1 = true'
 684         >>> rb.get_toml("key1")
 685         'true'
 686         """
 687
 688         cur_section = None
 689         for line in self.config_toml.splitlines():
 690             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 691             if section_match is not None:
 692                 cur_section = section_match.group(1)
 693
 694             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 695             if match is not None:
 696                 value = match.group(1)
 697                 if section is None or section == cur_section:
 698                     return self.get_string(value) or value.strip()
 699         return None
 700
 701     def cargo(self):
 702         """Return config path for cargo"""
 703         return self.program_config('cargo')
 704
 705     def rustc(self):
 706         """Return config path for rustc"""
 707         return self.program_config('rustc')
 708
 709     def rustfmt(self):
 710         """Return config path for rustfmt"""
 711         if not self.rustfmt_channel:
 712             return None
 713         return self.program_config('rustfmt')
 714
 715     def program_config(self, program):
 716         """Return config path for the given program
 717
 718         >>> rb = RustBuild()
 719         >>> rb.config_toml = 'rustc = "rustc"\\n'
 720         >>> rb.program_config('rustc')
 721         'rustc'
 722         >>> rb.config_toml = ''
 723         >>> cargo_path = rb.program_config('cargo')
 724         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 725         ... "bin", "cargo")
 726         True
 727         """
 728         config = self.get_toml(program)
 729         if config:
 730             return os.path.expanduser(config)
 731         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 732             program, self.exe_suffix()))
 733
 734     @staticmethod
 735     def get_string(line):
 736         """Return the value between double quotes
 737
 738         >>> RustBuild.get_string('    "devel"   ')
 739         'devel'
 740         >>> RustBuild.get_string("    'devel'   ")
 741         'devel'
 742         >>> RustBuild.get_string('devel') is None
 743         True
 744         >>> RustBuild.get_string('    "devel   ')
 745         ''
 746         """
 747         start = line.find('"')
 748         if start != -1:
 749             end = start + 1 + line[start + 1:].find('"')
 750             return line[start + 1:end]
 751         start = line.find('\'')
 752         if start != -1:
 753             end = start + 1 + line[start + 1:].find('\'')
 754             return line[start + 1:end]
 755         return None
 756
 757     @staticmethod
 758     def exe_suffix():
 759         """Return a suffix for executables"""
 760         if sys.platform == 'win32':
 761             return '.exe'
 762         return ''
 763
 764     def bootstrap_binary(self):
 765         """Return the path of the bootstrap binary
 766
 767         >>> rb = RustBuild()
 768         >>> rb.build_dir = "build"
 769         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 770         ... "debug", "bootstrap")
 771         True
 772         """
 773         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 774
 775     def build_bootstrap(self):
 776         """Build bootstrap"""
 777         build_dir = os.path.join(self.build_dir, "bootstrap")
 778         if self.clean and os.path.exists(build_dir):
 779             shutil.rmtree(build_dir)
 780         env = os.environ.copy()
 781         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 782         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 783         if "CARGO_BUILD_TARGET" in env:
 784             del env["CARGO_BUILD_TARGET"]
 785         env["CARGO_TARGET_DIR"] = build_dir
 786         env["RUSTC"] = self.rustc()
 787         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 788             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 789             if "LD_LIBRARY_PATH" in env else ""
 790         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 791             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 792             if "DYLD_LIBRARY_PATH" in env else ""
 793         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 794             (os.pathsep + env["LIBRARY_PATH"]) \
 795             if "LIBRARY_PATH" in env else ""
 796         # preserve existing RUSTFLAGS
 797         env.setdefault("RUSTFLAGS", "")
 798         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 799
 800         build_section = "target.{}".format(self.build)
 801         target_features = []
 802         if self.get_toml("crt-static", build_section) == "true":
 803             target_features += ["+crt-static"]
 804         elif self.get_toml("crt-static", build_section) == "false":
 805             target_features += ["-crt-static"]
 806         if target_features:
 807             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 808         target_linker = self.get_toml("linker", build_section)
 809         if target_linker is not None:
 810             env["RUSTFLAGS"] += " -C linker=" + target_linker
 811         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 812         if self.get_toml("deny-warnings", "rust") != "false":
 813             env["RUSTFLAGS"] += " -Dwarnings"
 814
 815         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 816             os.pathsep + env["PATH"]
 817         if not os.path.isfile(self.cargo()):
 818             raise Exception("no cargo executable found at `{}`".format(
 819                 self.cargo()))
 820         args = [self.cargo(), "build", "--manifest-path",
 821                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 822         for _ in range(1, self.verbose):
 823             args.append("--verbose")
 824         if self.use_locked_deps:
 825             args.append("--locked")
 826         if self.use_vendored_sources:
 827             args.append("--frozen")
 828         run(args, env=env, verbose=self.verbose)
 829
 830     def build_triple(self):
 831         """Build triple as in LLVM
 832
 833         Note that `default_build_triple` is moderately expensive,
 834         so use `self.build` where possible.
 835         """
 836         config = self.get_toml('build')
 837         if config:
 838             return config
 839         return default_build_triple(self.verbose)
 840
 841     def check_submodule(self, module, slow_submodules):
 842         if not slow_submodules:
 843             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 844                                            cwd=os.path.join(self.rust_root, module),
 845                                            stdout=subprocess.PIPE)
 846             return checked_out
 847         else:
 848             return None
 849
 850     def update_submodule(self, module, checked_out, recorded_submodules):
 851         module_path = os.path.join(self.rust_root, module)
 852
 853         if checked_out is not None:
 854             default_encoding = sys.getdefaultencoding()
 855             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 856             if recorded_submodules[module] == checked_out:
 857                 return
 858
 859         print("Updating submodule", module)
 860
 861         run(["git", "submodule", "-q", "sync", module],
 862             cwd=self.rust_root, verbose=self.verbose)
 863
 864         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 865         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 866             update_args.append("--progress")
 867         update_args.append(module)
 868         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 869
 870         run(["git", "reset", "-q", "--hard"],
 871             cwd=module_path, verbose=self.verbose)
 872         run(["git", "clean", "-qdfx"],
 873             cwd=module_path, verbose=self.verbose)
 874
 875     def update_submodules(self):
 876         """Update submodules"""
 877         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 878                 self.get_toml('submodules') == "false":
 879             return
 880
 881         default_encoding = sys.getdefaultencoding()
 882
 883         # check the existence and version of 'git' command
 884         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 885         self.git_version = distutils.version.LooseVersion(git_version_str)
 886
 887         slow_submodules = self.get_toml('fast-submodules') == "false"
 888         start_time = time()
 889         if slow_submodules:
 890             print('Unconditionally updating all submodules')
 891         else:
 892             print('Updating only changed submodules')
 893         default_encoding = sys.getdefaultencoding()
 894         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 895             ["git", "config", "--file",
 896              os.path.join(self.rust_root, ".gitmodules"),
 897              "--get-regexp", "path"]
 898         ).decode(default_encoding).splitlines()]
 899         filtered_submodules = []
 900         submodules_names = []
 901         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
 902         external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
 903         llvm_needed = not self.get_toml('codegen-backends', 'rust') \
 904             or "llvm" in self.get_toml('codegen-backends', 'rust')
 905         for module in submodules:
 906             if module.endswith("llvm-project"):
 907                 # Don't sync the llvm-project submodule if an external LLVM was
 908                 # provided, if we are downloading LLVM or if the LLVM backend is
 909                 # not being built. Also, if the submodule has been initialized
 910                 # already, sync it anyways so that it doesn't mess up contributor
 911                 # pull requests.
 912                 if external_llvm_provided or not llvm_needed:
 913                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
 914                         continue
 915             check = self.check_submodule(module, slow_submodules)
 916             filtered_submodules.append((module, check))
 917             submodules_names.append(module)
 918         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 919                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 920         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 921         recorded_submodules = {}
 922         for data in recorded:
 923             data = data.split()
 924             recorded_submodules[data[3]] = data[2]
 925         for module in filtered_submodules:
 926             self.update_submodule(module[0], module[1], recorded_submodules)
 927         print("Submodules updated in %.2f seconds" % (time() - start_time))
 928
 929     def set_normal_environment(self):
 930         """Set download URL for normal environment"""
 931         if 'RUSTUP_DIST_SERVER' in os.environ:
 932             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 933         else:
 934             self._download_url = 'https://static.rust-lang.org'
 935
 936     def set_dev_environment(self):
 937         """Set download URL for development environment"""
 938         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 939             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 940         else:
 941             self._download_url = 'https://dev-static.rust-lang.org'
 942
 943     def check_vendored_status(self):
 944         """Check that vendoring is configured properly"""
 945         vendor_dir = os.path.join(self.rust_root, 'vendor')
 946         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 947             if os.environ.get('USER') != os.environ['SUDO_USER']:
 948                 self.use_vendored_sources = True
 949                 print('info: looks like you are running this command under `sudo`')
 950                 print('      and so in order to preserve your $HOME this will now')
 951                 print('      use vendored sources by default.')
 952                 if not os.path.exists(vendor_dir):
 953                     print('error: vendoring required, but vendor directory does not exist.')
 954                     print('       Run `cargo vendor` without sudo to initialize the '
 955                           'vendor directory.')
 956                     raise Exception("{} not found".format(vendor_dir))
 957
 958         if self.use_vendored_sources:
 959             if not os.path.exists('.cargo'):
 960                 os.makedirs('.cargo')
 961             with output('.cargo/config') as cargo_config:
 962                 cargo_config.write(
 963                     "[source.crates-io]\n"
 964                     "replace-with = 'vendored-sources'\n"
 965                     "registry = 'https://example.com'\n"
 966                     "\n"
 967                     "[source.vendored-sources]\n"
 968                     "directory = '{}/vendor'\n"
 969                     .format(self.rust_root))
 970         else:
 971             if os.path.exists('.cargo'):
 972                 shutil.rmtree('.cargo')
 973
 974     def ensure_vendored(self):
 975         """Ensure that the vendored sources are available if needed"""
 976         vendor_dir = os.path.join(self.rust_root, 'vendor')
 977         # Note that this does not handle updating the vendored dependencies if
 978         # the rust git repository is updated. Normal development usually does
 979         # not use vendoring, so hopefully this isn't too much of a problem.
 980         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 981             run([
 982                 self.cargo(),
 983                 "vendor",
 984                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
 985                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
 986             ], verbose=self.verbose, cwd=self.rust_root)
 987
 988
 989 def bootstrap(help_triggered):
 990     """Configure, fetch, build and run the initial bootstrap"""
 991
 992     # If the user is asking for help, let them know that the whole download-and-build
 993     # process has to happen before anything is printed out.
 994     if help_triggered:
 995         print("info: Downloading and building bootstrap before processing --help")
 996         print("      command. See src/bootstrap/README.md for help with common")
 997         print("      commands.")
 998
 999     parser = argparse.ArgumentParser(description='Build rust')
1000     parser.add_argument('--config')
1001     parser.add_argument('--build')
1002     parser.add_argument('--clean', action='store_true')
1003     parser.add_argument('-v', '--verbose', action='count', default=0)
1004
1005     args = [a for a in sys.argv if a != '-h' and a != '--help']
1006     args, _ = parser.parse_known_args(args)
1007
1008     # Configure initial bootstrap
1009     build = RustBuild()
1010     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1011     build.verbose = args.verbose
1012     build.clean = args.clean
1013
1014     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1015     # exists).
1016     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1017     if not toml_path and os.path.exists('config.toml'):
1018         toml_path = 'config.toml'
1019
1020     if toml_path:
1021         if not os.path.exists(toml_path):
1022             toml_path = os.path.join(build.rust_root, toml_path)
1023
1024         with open(toml_path) as config:
1025             build.config_toml = config.read()
1026
1027     profile = build.get_toml('profile')
1028     if profile is not None:
1029         include_file = 'config.{}.toml'.format(profile)
1030         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1031         include_path = os.path.join(include_dir, include_file)
1032         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1033         # specific key, so appending our defaults at the end allows the user to override them
1034         with open(include_path) as included_toml:
1035             build.config_toml += os.linesep + included_toml.read()
1036
1037     config_verbose = build.get_toml('verbose', 'build')
1038     if config_verbose is not None:
1039         build.verbose = max(build.verbose, int(config_verbose))
1040
1041     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1042
1043     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1044
1045     build.check_vendored_status()
1046
1047     build_dir = build.get_toml('build-dir', 'build') or 'build'
1048     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1049
1050     data = stage0_data(build.rust_root)
1051     build.date = data['date']
1052     build.rustc_channel = data['rustc']
1053
1054     if "rustfmt" in data:
1055         build.rustfmt_channel = data['rustfmt']
1056
1057     if 'dev' in data:
1058         build.set_dev_environment()
1059     else:
1060         build.set_normal_environment()
1061
1062     build.update_submodules()
1063
1064     # Fetch/build the bootstrap
1065     build.build = args.build or build.build_triple()
1066     build.download_stage0()
1067     sys.stdout.flush()
1068     build.ensure_vendored()
1069     build.build_bootstrap()
1070     sys.stdout.flush()
1071
1072     # Run the bootstrap
1073     args = [build.bootstrap_binary()]
1074     args.extend(sys.argv[1:])
1075     env = os.environ.copy()
1076     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1077     env["BOOTSTRAP_PYTHON"] = sys.executable
1078     env["BUILD_DIR"] = build.build_dir
1079     env["RUSTC_BOOTSTRAP"] = '1'
1080     if toml_path:
1081         env["BOOTSTRAP_CONFIG"] = toml_path
1082     run(args, env=env, verbose=build.verbose)
1083
1084
1085 def main():
1086     """Entry point for the bootstrap process"""
1087     start_time = time()
1088
1089     # x.py help <cmd> ...
1090     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1091         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1092
1093     help_triggered = (
1094         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1095     try:
1096         bootstrap(help_triggered)
1097         if not help_triggered:
1098             print("Build completed successfully in {}".format(
1099                 format_build_time(time() - start_time)))
1100     except (SystemExit, KeyboardInterrupt) as error:
1101         if hasattr(error, 'code') and isinstance(error.code, int):
1102             exit_code = error.code
1103         else:
1104             exit_code = 1
1105             print(error)
1106         if not help_triggered:
1107             print("Build completed unsuccessfully in {}".format(
1108                 format_build_time(time() - start_time)))
1109         sys.exit(exit_code)
1110
1111
1112 if __name__ == '__main__':
1113     main()