src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple():
 191     """Build triple as in LLVM"""
 192     default_encoding = sys.getdefaultencoding()
 193     required = sys.platform != 'win32'
 194     ostype = require(["uname", "-s"], exit=required)
 195     cputype = require(['uname', '-m'], exit=required)
 196
 197     # If we do not have `uname`, assume Windows.
 198     if ostype is None or cputype is None:
 199         return 'x86_64-pc-windows-msvc'
 200
 201     ostype = ostype.decode(default_encoding)
 202     cputype = cputype.decode(default_encoding)
 203
 204     # The goal here is to come up with the same triple as LLVM would,
 205     # at least for the subset of platforms we're willing to target.
 206     ostype_mapper = {
 207         'Darwin': 'apple-darwin',
 208         'DragonFly': 'unknown-dragonfly',
 209         'FreeBSD': 'unknown-freebsd',
 210         'Haiku': 'unknown-haiku',
 211         'NetBSD': 'unknown-netbsd',
 212         'OpenBSD': 'unknown-openbsd'
 213     }
 214
 215     # Consider the direct transformation first and then the special cases
 216     if ostype in ostype_mapper:
 217         ostype = ostype_mapper[ostype]
 218     elif ostype == 'Linux':
 219         os_from_sp = subprocess.check_output(
 220             ['uname', '-o']).strip().decode(default_encoding)
 221         if os_from_sp == 'Android':
 222             ostype = 'linux-android'
 223         else:
 224             ostype = 'unknown-linux-gnu'
 225     elif ostype == 'SunOS':
 226         ostype = 'sun-solaris'
 227         # On Solaris, uname -m will return a machine classification instead
 228         # of a cpu type, so uname -p is recommended instead.  However, the
 229         # output from that option is too generic for our purposes (it will
 230         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 231         # must be used instead.
 232         cputype = require(['isainfo', '-k']).decode(default_encoding)
 233     elif ostype.startswith('MINGW'):
 234         # msys' `uname` does not print gcc configuration, but prints msys
 235         # configuration. so we cannot believe `uname -m`:
 236         # msys1 is always i686 and msys2 is always x86_64.
 237         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 238         # MINGW64 on x86_64.
 239         ostype = 'pc-windows-gnu'
 240         cputype = 'i686'
 241         if os.environ.get('MSYSTEM') == 'MINGW64':
 242             cputype = 'x86_64'
 243     elif ostype.startswith('MSYS'):
 244         ostype = 'pc-windows-gnu'
 245     elif ostype.startswith('CYGWIN_NT'):
 246         cputype = 'i686'
 247         if ostype.endswith('WOW64'):
 248             cputype = 'x86_64'
 249         ostype = 'pc-windows-gnu'
 250     elif sys.platform == 'win32':
 251         # Some Windows platforms might have a `uname` command that returns a
 252         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 253         # these cases, fall back to using sys.platform.
 254         return 'x86_64-pc-windows-msvc'
 255     else:
 256         err = "unknown OS type: {}".format(ostype)
 257         sys.exit(err)
 258
 259     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 260         cputype = subprocess.check_output(
 261               ['uname', '-p']).strip().decode(default_encoding)
 262     cputype_mapper = {
 263         'BePC': 'i686',
 264         'aarch64': 'aarch64',
 265         'amd64': 'x86_64',
 266         'arm64': 'aarch64',
 267         'i386': 'i686',
 268         'i486': 'i686',
 269         'i686': 'i686',
 270         'i786': 'i686',
 271         'powerpc': 'powerpc',
 272         'powerpc64': 'powerpc64',
 273         'powerpc64le': 'powerpc64le',
 274         'ppc': 'powerpc',
 275         'ppc64': 'powerpc64',
 276         'ppc64le': 'powerpc64le',
 277         's390x': 's390x',
 278         'x64': 'x86_64',
 279         'x86': 'i686',
 280         'x86-64': 'x86_64',
 281         'x86_64': 'x86_64'
 282     }
 283
 284     # Consider the direct transformation first and then the special cases
 285     if cputype in cputype_mapper:
 286         cputype = cputype_mapper[cputype]
 287     elif cputype in {'xscale', 'arm'}:
 288         cputype = 'arm'
 289         if ostype == 'linux-android':
 290             ostype = 'linux-androideabi'
 291         elif ostype == 'unknown-freebsd':
 292             cputype = subprocess.check_output(
 293                 ['uname', '-p']).strip().decode(default_encoding)
 294             ostype = 'unknown-freebsd'
 295     elif cputype == 'armv6l':
 296         cputype = 'arm'
 297         if ostype == 'linux-android':
 298             ostype = 'linux-androideabi'
 299         else:
 300             ostype += 'eabihf'
 301     elif cputype in {'armv7l', 'armv8l'}:
 302         cputype = 'armv7'
 303         if ostype == 'linux-android':
 304             ostype = 'linux-androideabi'
 305         else:
 306             ostype += 'eabihf'
 307     elif cputype == 'mips':
 308         if sys.byteorder == 'big':
 309             cputype = 'mips'
 310         elif sys.byteorder == 'little':
 311             cputype = 'mipsel'
 312         else:
 313             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 314     elif cputype == 'mips64':
 315         if sys.byteorder == 'big':
 316             cputype = 'mips64'
 317         elif sys.byteorder == 'little':
 318             cputype = 'mips64el'
 319         else:
 320             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 321         # only the n64 ABI is supported, indicate it
 322         ostype += 'abi64'
 323     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 324         pass
 325     else:
 326         err = "unknown cpu type: {}".format(cputype)
 327         sys.exit(err)
 328
 329     return "{}-{}".format(cputype, ostype)
 330
 331
 332 @contextlib.contextmanager
 333 def output(filepath):
 334     tmp = filepath + '.tmp'
 335     with open(tmp, 'w') as f:
 336         yield f
 337     try:
 338         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 339         os.rename(tmp, filepath)
 340     except OSError:
 341         shutil.copy2(tmp, filepath)
 342         os.remove(tmp)
 343
 344
 345 class RustBuild(object):
 346     """Provide all the methods required to build Rust"""
 347     def __init__(self):
 348         self.cargo_channel = ''
 349         self.date = ''
 350         self._download_url = ''
 351         self.rustc_channel = ''
 352         self.rustfmt_channel = ''
 353         self.build = ''
 354         self.build_dir = ''
 355         self.clean = False
 356         self.config_toml = ''
 357         self.rust_root = ''
 358         self.use_locked_deps = ''
 359         self.use_vendored_sources = ''
 360         self.verbose = False
 361         self.git_version = None
 362         self.nix_deps_dir = None
 363
 364     def download_stage0(self):
 365         """Fetch the build system for Rust, written in Rust
 366
 367         This method will build a cache directory, then it will fetch the
 368         tarball which has the stage0 compiler used to then bootstrap the Rust
 369         compiler itself.
 370
 371         Each downloaded tarball is extracted, after that, the script
 372         will move all the content to the right place.
 373         """
 374         rustc_channel = self.rustc_channel
 375         cargo_channel = self.cargo_channel
 376         rustfmt_channel = self.rustfmt_channel
 377
 378         if self.rustc().startswith(self.bin_root()) and \
 379                 (not os.path.exists(self.rustc()) or
 380                  self.program_out_of_date(self.rustc_stamp())):
 381             if os.path.exists(self.bin_root()):
 382                 shutil.rmtree(self.bin_root())
 383             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 384             filename = "rust-std-{}-{}{}".format(
 385                 rustc_channel, self.build, tarball_suffix)
 386             pattern = "rust-std-{}".format(self.build)
 387             self._download_stage0_helper(filename, pattern, tarball_suffix)
 388
 389             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 392             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 393             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 394             lib_dir = "{}/lib".format(self.bin_root())
 395             for lib in os.listdir(lib_dir):
 396                 if lib.endswith(".so"):
 397                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 398             with output(self.rustc_stamp()) as rust_stamp:
 399                 rust_stamp.write(self.date)
 400
 401         if self.cargo().startswith(self.bin_root()) and \
 402                 (not os.path.exists(self.cargo()) or
 403                  self.program_out_of_date(self.cargo_stamp())):
 404             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 405             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 406                                               tarball_suffix)
 407             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 408             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 409             with output(self.cargo_stamp()) as cargo_stamp:
 410                 cargo_stamp.write(self.date)
 411
 412         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 413             not os.path.exists(self.rustfmt())
 414             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 415         ):
 416             if rustfmt_channel:
 417                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 418                 [channel, date] = rustfmt_channel.split('-', 1)
 419                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 420                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 421                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 422                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 423                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 424                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 425
 426         if self.downloading_llvm():
 427             # We want the most recent LLVM submodule update to avoid downloading
 428             # LLVM more often than necessary.
 429             #
 430             # This git command finds that commit SHA, looking for bors-authored
 431             # merges that modified src/llvm-project.
 432             #
 433             # This works even in a repository that has not yet initialized
 434             # submodules.
 435             llvm_sha = subprocess.check_output([
 436                 "git", "log", "--author=bors", "--format=%H", "-n1",
 437                 "-m", "--first-parent",
 438                 "--",
 439                 "src/llvm-project",
 440                 "src/bootstrap/download-ci-llvm-stamp",
 441             ]).decode(sys.getdefaultencoding()).strip()
 442             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 443             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 444                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 445                 for binary in ["llvm-config", "FileCheck"]:
 446                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 447                 with output(self.llvm_stamp()) as llvm_stamp:
 448                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 449
 450     def downloading_llvm(self):
 451         opt = self.get_toml('download-ci-llvm', 'llvm')
 452         return opt == "true" \
 453             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 454
 455     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 456         if date is None:
 457             date = self.date
 458         cache_dst = os.path.join(self.build_dir, "cache")
 459         rustc_cache = os.path.join(cache_dst, date)
 460         if not os.path.exists(rustc_cache):
 461             os.makedirs(rustc_cache)
 462
 463         url = "{}/dist/{}".format(self._download_url, date)
 464         tarball = os.path.join(rustc_cache, filename)
 465         if not os.path.exists(tarball):
 466             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 467         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 468
 469     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 470         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 471         cache_dst = os.path.join(self.build_dir, "cache")
 472         rustc_cache = os.path.join(cache_dst, cache_prefix)
 473         if not os.path.exists(rustc_cache):
 474             os.makedirs(rustc_cache)
 475
 476         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 477         if llvm_assertions:
 478             url = url.replace('rustc-builds', 'rustc-builds-alt')
 479         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 480         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 481         tarball = os.path.join(rustc_cache, filename)
 482         if not os.path.exists(tarball):
 483             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 484         unpack(tarball, tarball_suffix, self.llvm_root(),
 485                 match="rust-dev",
 486                 verbose=self.verbose)
 487
 488     def fix_bin_or_dylib(self, fname):
 489         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 490         or the RPATH section, to fix the dynamic library search path
 491
 492         This method is only required on NixOS and uses the PatchELF utility to
 493         change the interpreter/RPATH of ELF executables.
 494
 495         Please see https://nixos.org/patchelf.html for more information
 496         """
 497         default_encoding = sys.getdefaultencoding()
 498         try:
 499             ostype = subprocess.check_output(
 500                 ['uname', '-s']).strip().decode(default_encoding)
 501         except subprocess.CalledProcessError:
 502             return
 503         except OSError as reason:
 504             if getattr(reason, 'winerror', None) is not None:
 505                 return
 506             raise reason
 507
 508         if ostype != "Linux":
 509             return
 510
 511         if not os.path.exists("/etc/NIXOS"):
 512             return
 513         if os.path.exists("/lib"):
 514             return
 515
 516         # At this point we're pretty sure the user is running NixOS
 517         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 518         print(nix_os_msg, fname)
 519
 520         # Only build `stage0/.nix-deps` once.
 521         nix_deps_dir = self.nix_deps_dir
 522         if not nix_deps_dir:
 523             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 524             if not os.path.exists(nix_deps_dir):
 525                 os.makedirs(nix_deps_dir)
 526
 527             nix_deps = [
 528                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 529                 "stdenv.cc.bintools",
 530
 531                 # Needed as a system dependency of `libLLVM-*.so`.
 532                 "zlib",
 533
 534                 # Needed for patching ELF binaries (see doc comment above).
 535                 "patchelf",
 536             ]
 537
 538             # Run `nix-build` to "build" each dependency (which will likely reuse
 539             # the existing `/nix/store` copy, or at most download a pre-built copy).
 540             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 541             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 542             # ensuring garbage collection will never remove the `/nix/store` path
 543             # (which would break our patched binaries that hardcode those paths).
 544             for dep in nix_deps:
 545                 try:
 546                     subprocess.check_output([
 547                         "nix-build", "<nixpkgs>",
 548                         "-A", dep,
 549                         "-o", "{}/{}".format(nix_deps_dir, dep),
 550                     ])
 551                 except subprocess.CalledProcessError as reason:
 552                     print("warning: failed to call nix-build:", reason)
 553                     return
 554
 555             self.nix_deps_dir = nix_deps_dir
 556
 557         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 558
 559         if fname.endswith(".so"):
 560             # Dynamic library, patch RPATH to point to system dependencies.
 561             dylib_deps = ["zlib"]
 562             rpath_entries = [
 563                 # Relative default, all binary and dynamic libraries we ship
 564                 # appear to have this (even when `../lib` is redundant).
 565                 "$ORIGIN/../lib",
 566             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 567             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 568         else:
 569             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 570             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 571                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 572
 573         try:
 574             subprocess.check_output([patchelf] + patchelf_args + [fname])
 575         except subprocess.CalledProcessError as reason:
 576             print("warning: failed to call patchelf:", reason)
 577             return
 578
 579     def rustc_stamp(self):
 580         """Return the path for .rustc-stamp
 581
 582         >>> rb = RustBuild()
 583         >>> rb.build_dir = "build"
 584         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 585         True
 586         """
 587         return os.path.join(self.bin_root(), '.rustc-stamp')
 588
 589     def cargo_stamp(self):
 590         """Return the path for .cargo-stamp
 591
 592         >>> rb = RustBuild()
 593         >>> rb.build_dir = "build"
 594         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 595         True
 596         """
 597         return os.path.join(self.bin_root(), '.cargo-stamp')
 598
 599     def rustfmt_stamp(self):
 600         """Return the path for .rustfmt-stamp
 601
 602         >>> rb = RustBuild()
 603         >>> rb.build_dir = "build"
 604         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 605         True
 606         """
 607         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 608
 609     def llvm_stamp(self):
 610         """Return the path for .rustfmt-stamp
 611
 612         >>> rb = RustBuild()
 613         >>> rb.build_dir = "build"
 614         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 615         True
 616         """
 617         return os.path.join(self.llvm_root(), '.llvm-stamp')
 618
 619
 620     def program_out_of_date(self, stamp_path, extra=""):
 621         """Check if the given program stamp is out of date"""
 622         if not os.path.exists(stamp_path) or self.clean:
 623             return True
 624         with open(stamp_path, 'r') as stamp:
 625             return (self.date + extra) != stamp.read()
 626
 627     def bin_root(self):
 628         """Return the binary root directory
 629
 630         >>> rb = RustBuild()
 631         >>> rb.build_dir = "build"
 632         >>> rb.bin_root() == os.path.join("build", "stage0")
 633         True
 634
 635         When the 'build' property is given should be a nested directory:
 636
 637         >>> rb.build = "devel"
 638         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 639         True
 640         """
 641         return os.path.join(self.build_dir, self.build, "stage0")
 642
 643     def llvm_root(self):
 644         """Return the CI LLVM root directory
 645
 646         >>> rb = RustBuild()
 647         >>> rb.build_dir = "build"
 648         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 649         True
 650
 651         When the 'build' property is given should be a nested directory:
 652
 653         >>> rb.build = "devel"
 654         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 655         True
 656         """
 657         return os.path.join(self.build_dir, self.build, "ci-llvm")
 658
 659     def get_toml(self, key, section=None):
 660         """Returns the value of the given key in config.toml, otherwise returns None
 661
 662         >>> rb = RustBuild()
 663         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 664         >>> rb.get_toml("key2")
 665         'value2'
 666
 667         If the key does not exists, the result is None:
 668
 669         >>> rb.get_toml("key3") is None
 670         True
 671
 672         Optionally also matches the section the key appears in
 673
 674         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 675         >>> rb.get_toml('key', 'a')
 676         'value1'
 677         >>> rb.get_toml('key', 'b')
 678         'value2'
 679         >>> rb.get_toml('key', 'c') is None
 680         True
 681
 682         >>> rb.config_toml = 'key1 = true'
 683         >>> rb.get_toml("key1")
 684         'true'
 685         """
 686
 687         cur_section = None
 688         for line in self.config_toml.splitlines():
 689             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 690             if section_match is not None:
 691                 cur_section = section_match.group(1)
 692
 693             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 694             if match is not None:
 695                 value = match.group(1)
 696                 if section is None or section == cur_section:
 697                     return self.get_string(value) or value.strip()
 698         return None
 699
 700     def cargo(self):
 701         """Return config path for cargo"""
 702         return self.program_config('cargo')
 703
 704     def rustc(self):
 705         """Return config path for rustc"""
 706         return self.program_config('rustc')
 707
 708     def rustfmt(self):
 709         """Return config path for rustfmt"""
 710         if not self.rustfmt_channel:
 711             return None
 712         return self.program_config('rustfmt')
 713
 714     def program_config(self, program):
 715         """Return config path for the given program
 716
 717         >>> rb = RustBuild()
 718         >>> rb.config_toml = 'rustc = "rustc"\\n'
 719         >>> rb.program_config('rustc')
 720         'rustc'
 721         >>> rb.config_toml = ''
 722         >>> cargo_path = rb.program_config('cargo')
 723         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 724         ... "bin", "cargo")
 725         True
 726         """
 727         config = self.get_toml(program)
 728         if config:
 729             return os.path.expanduser(config)
 730         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 731             program, self.exe_suffix()))
 732
 733     @staticmethod
 734     def get_string(line):
 735         """Return the value between double quotes
 736
 737         >>> RustBuild.get_string('    "devel"   ')
 738         'devel'
 739         >>> RustBuild.get_string("    'devel'   ")
 740         'devel'
 741         >>> RustBuild.get_string('devel') is None
 742         True
 743         >>> RustBuild.get_string('    "devel   ')
 744         ''
 745         """
 746         start = line.find('"')
 747         if start != -1:
 748             end = start + 1 + line[start + 1:].find('"')
 749             return line[start + 1:end]
 750         start = line.find('\'')
 751         if start != -1:
 752             end = start + 1 + line[start + 1:].find('\'')
 753             return line[start + 1:end]
 754         return None
 755
 756     @staticmethod
 757     def exe_suffix():
 758         """Return a suffix for executables"""
 759         if sys.platform == 'win32':
 760             return '.exe'
 761         return ''
 762
 763     def bootstrap_binary(self):
 764         """Return the path of the bootstrap binary
 765
 766         >>> rb = RustBuild()
 767         >>> rb.build_dir = "build"
 768         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 769         ... "debug", "bootstrap")
 770         True
 771         """
 772         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 773
 774     def build_bootstrap(self):
 775         """Build bootstrap"""
 776         build_dir = os.path.join(self.build_dir, "bootstrap")
 777         if self.clean and os.path.exists(build_dir):
 778             shutil.rmtree(build_dir)
 779         env = os.environ.copy()
 780         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 781         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 782         if "CARGO_BUILD_TARGET" in env:
 783             del env["CARGO_BUILD_TARGET"]
 784         env["CARGO_TARGET_DIR"] = build_dir
 785         env["RUSTC"] = self.rustc()
 786         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 787             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 788             if "LD_LIBRARY_PATH" in env else ""
 789         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 790             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 791             if "DYLD_LIBRARY_PATH" in env else ""
 792         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 793             (os.pathsep + env["LIBRARY_PATH"]) \
 794             if "LIBRARY_PATH" in env else ""
 795         # preserve existing RUSTFLAGS
 796         env.setdefault("RUSTFLAGS", "")
 797         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 798
 799         build_section = "target.{}".format(self.build_triple())
 800         target_features = []
 801         if self.get_toml("crt-static", build_section) == "true":
 802             target_features += ["+crt-static"]
 803         elif self.get_toml("crt-static", build_section) == "false":
 804             target_features += ["-crt-static"]
 805         if target_features:
 806             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 807         target_linker = self.get_toml("linker", build_section)
 808         if target_linker is not None:
 809             env["RUSTFLAGS"] += " -C linker=" + target_linker
 810         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 811         if self.get_toml("deny-warnings", "rust") != "false":
 812             env["RUSTFLAGS"] += " -Dwarnings"
 813
 814         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 815             os.pathsep + env["PATH"]
 816         if not os.path.isfile(self.cargo()):
 817             raise Exception("no cargo executable found at `{}`".format(
 818                 self.cargo()))
 819         args = [self.cargo(), "build", "--manifest-path",
 820                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 821         for _ in range(1, self.verbose):
 822             args.append("--verbose")
 823         if self.use_locked_deps:
 824             args.append("--locked")
 825         if self.use_vendored_sources:
 826             args.append("--frozen")
 827         run(args, env=env, verbose=self.verbose)
 828
 829     def build_triple(self):
 830         """Build triple as in LLVM"""
 831         config = self.get_toml('build')
 832         if config:
 833             return config
 834         return default_build_triple()
 835
 836     def check_submodule(self, module, slow_submodules):
 837         if not slow_submodules:
 838             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 839                                            cwd=os.path.join(self.rust_root, module),
 840                                            stdout=subprocess.PIPE)
 841             return checked_out
 842         else:
 843             return None
 844
 845     def update_submodule(self, module, checked_out, recorded_submodules):
 846         module_path = os.path.join(self.rust_root, module)
 847
 848         if checked_out is not None:
 849             default_encoding = sys.getdefaultencoding()
 850             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 851             if recorded_submodules[module] == checked_out:
 852                 return
 853
 854         print("Updating submodule", module)
 855
 856         run(["git", "submodule", "-q", "sync", module],
 857             cwd=self.rust_root, verbose=self.verbose)
 858
 859         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 860         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 861             update_args.append("--progress")
 862         update_args.append(module)
 863         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 864
 865         run(["git", "reset", "-q", "--hard"],
 866             cwd=module_path, verbose=self.verbose)
 867         run(["git", "clean", "-qdfx"],
 868             cwd=module_path, verbose=self.verbose)
 869
 870     def update_submodules(self):
 871         """Update submodules"""
 872         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 873                 self.get_toml('submodules') == "false":
 874             return
 875
 876         default_encoding = sys.getdefaultencoding()
 877
 878         # check the existence and version of 'git' command
 879         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 880         self.git_version = distutils.version.LooseVersion(git_version_str)
 881
 882         slow_submodules = self.get_toml('fast-submodules') == "false"
 883         start_time = time()
 884         if slow_submodules:
 885             print('Unconditionally updating all submodules')
 886         else:
 887             print('Updating only changed submodules')
 888         default_encoding = sys.getdefaultencoding()
 889         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 890             ["git", "config", "--file",
 891              os.path.join(self.rust_root, ".gitmodules"),
 892              "--get-regexp", "path"]
 893         ).decode(default_encoding).splitlines()]
 894         filtered_submodules = []
 895         submodules_names = []
 896         llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
 897         for module in submodules:
 898             if module.endswith("llvm-project"):
 899                 # Don't sync the llvm-project submodule either if an external LLVM
 900                 # was provided, or if we are downloading LLVM. Also, if the
 901                 # submodule has been initialized already, sync it anyways so that
 902                 # it doesn't mess up contributor pull requests.
 903                 if self.get_toml('llvm-config') or self.downloading_llvm():
 904                     if self.get_toml('lld') != 'true' and not llvm_checked_out:
 905                         continue
 906             check = self.check_submodule(module, slow_submodules)
 907             filtered_submodules.append((module, check))
 908             submodules_names.append(module)
 909         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 910                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 911         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 912         recorded_submodules = {}
 913         for data in recorded:
 914             data = data.split()
 915             recorded_submodules[data[3]] = data[2]
 916         for module in filtered_submodules:
 917             self.update_submodule(module[0], module[1], recorded_submodules)
 918         print("Submodules updated in %.2f seconds" % (time() - start_time))
 919
 920     def set_normal_environment(self):
 921         """Set download URL for normal environment"""
 922         if 'RUSTUP_DIST_SERVER' in os.environ:
 923             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 924         else:
 925             self._download_url = 'https://static.rust-lang.org'
 926
 927     def set_dev_environment(self):
 928         """Set download URL for development environment"""
 929         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 930             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 931         else:
 932             self._download_url = 'https://dev-static.rust-lang.org'
 933
 934     def check_vendored_status(self):
 935         """Check that vendoring is configured properly"""
 936         vendor_dir = os.path.join(self.rust_root, 'vendor')
 937         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 938             if os.environ.get('USER') != os.environ['SUDO_USER']:
 939                 self.use_vendored_sources = True
 940                 print('info: looks like you are running this command under `sudo`')
 941                 print('      and so in order to preserve your $HOME this will now')
 942                 print('      use vendored sources by default.')
 943                 if not os.path.exists(vendor_dir):
 944                     print('error: vendoring required, but vendor directory does not exist.')
 945                     print('       Run `cargo vendor` without sudo to initialize the '
 946                           'vendor directory.')
 947                     raise Exception("{} not found".format(vendor_dir))
 948
 949         if self.use_vendored_sources:
 950             if not os.path.exists('.cargo'):
 951                 os.makedirs('.cargo')
 952             with output('.cargo/config') as cargo_config:
 953                 cargo_config.write(
 954                     "[source.crates-io]\n"
 955                     "replace-with = 'vendored-sources'\n"
 956                     "registry = 'https://example.com'\n"
 957                     "\n"
 958                     "[source.vendored-sources]\n"
 959                     "directory = '{}/vendor'\n"
 960                     .format(self.rust_root))
 961         else:
 962             if os.path.exists('.cargo'):
 963                 shutil.rmtree('.cargo')
 964
 965     def ensure_vendored(self):
 966         """Ensure that the vendored sources are available if needed"""
 967         vendor_dir = os.path.join(self.rust_root, 'vendor')
 968         # Note that this does not handle updating the vendored dependencies if
 969         # the rust git repository is updated. Normal development usually does
 970         # not use vendoring, so hopefully this isn't too much of a problem.
 971         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 972             run([
 973                 self.cargo(),
 974                 "vendor",
 975                 "--sync=./src/tools/rust-analyzer/Cargo.toml",
 976                 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
 977             ], verbose=self.verbose, cwd=self.rust_root)
 978
 979
 980 def bootstrap(help_triggered):
 981     """Configure, fetch, build and run the initial bootstrap"""
 982
 983     # If the user is asking for help, let them know that the whole download-and-build
 984     # process has to happen before anything is printed out.
 985     if help_triggered:
 986         print("info: Downloading and building bootstrap before processing --help")
 987         print("      command. See src/bootstrap/README.md for help with common")
 988         print("      commands.")
 989
 990     parser = argparse.ArgumentParser(description='Build rust')
 991     parser.add_argument('--config')
 992     parser.add_argument('--build')
 993     parser.add_argument('--clean', action='store_true')
 994     parser.add_argument('-v', '--verbose', action='count', default=0)
 995
 996     args = [a for a in sys.argv if a != '-h' and a != '--help']
 997     args, _ = parser.parse_known_args(args)
 998
 999     # Configure initial bootstrap
1000     build = RustBuild()
1001     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1002     build.verbose = args.verbose
1003     build.clean = args.clean
1004
1005     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1006     # exists).
1007     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1008     if not toml_path and os.path.exists('config.toml'):
1009         toml_path = 'config.toml'
1010
1011     if toml_path:
1012         if not os.path.exists(toml_path):
1013             toml_path = os.path.join(build.rust_root, toml_path)
1014
1015         with open(toml_path) as config:
1016             build.config_toml = config.read()
1017
1018     profile = build.get_toml('profile')
1019     if profile is not None:
1020         include_file = 'config.{}.toml'.format(profile)
1021         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1022         include_path = os.path.join(include_dir, include_file)
1023         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1024         # specific key, so appending our defaults at the end allows the user to override them
1025         with open(include_path) as included_toml:
1026             build.config_toml += os.linesep + included_toml.read()
1027
1028     config_verbose = build.get_toml('verbose', 'build')
1029     if config_verbose is not None:
1030         build.verbose = max(build.verbose, int(config_verbose))
1031
1032     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1033
1034     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1035
1036     build.check_vendored_status()
1037
1038     build_dir = build.get_toml('build-dir', 'build') or 'build'
1039     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1040
1041     data = stage0_data(build.rust_root)
1042     build.date = data['date']
1043     build.rustc_channel = data['rustc']
1044     build.cargo_channel = data['cargo']
1045
1046     if "rustfmt" in data:
1047         build.rustfmt_channel = data['rustfmt']
1048
1049     if 'dev' in data:
1050         build.set_dev_environment()
1051     else:
1052         build.set_normal_environment()
1053
1054     build.update_submodules()
1055
1056     # Fetch/build the bootstrap
1057     build.build = args.build or build.build_triple()
1058     build.download_stage0()
1059     sys.stdout.flush()
1060     build.ensure_vendored()
1061     build.build_bootstrap()
1062     sys.stdout.flush()
1063
1064     # Run the bootstrap
1065     args = [build.bootstrap_binary()]
1066     args.extend(sys.argv[1:])
1067     env = os.environ.copy()
1068     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1069     env["BOOTSTRAP_PYTHON"] = sys.executable
1070     env["BUILD_DIR"] = build.build_dir
1071     env["RUSTC_BOOTSTRAP"] = '1'
1072     if toml_path:
1073         env["BOOTSTRAP_CONFIG"] = toml_path
1074     run(args, env=env, verbose=build.verbose)
1075
1076
1077 def main():
1078     """Entry point for the bootstrap process"""
1079     start_time = time()
1080
1081     # x.py help <cmd> ...
1082     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1083         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1084
1085     help_triggered = (
1086         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1087     try:
1088         bootstrap(help_triggered)
1089         if not help_triggered:
1090             print("Build completed successfully in {}".format(
1091                 format_build_time(time() - start_time)))
1092     except (SystemExit, KeyboardInterrupt) as error:
1093         if hasattr(error, 'code') and isinstance(error.code, int):
1094             exit_code = error.code
1095         else:
1096             exit_code = 1
1097             print(error)
1098         if not help_triggered:
1099             print("Build completed unsuccessfully in {}".format(
1100                 format_build_time(time() - start_time)))
1101         sys.exit(exit_code)
1102
1103
1104 if __name__ == '__main__':
1105     main()