src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17 def support_xz():
  18     try:
  19         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  20             temp_path = temp_file.name
  21         with tarfile.open(temp_path, "w:xz"):
  22             pass
  23         return True
  24     except tarfile.CompressionError:
  25         return False
  26
  27 def get(url, path, verbose=False, do_verify=True):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         if do_verify:
  37             download(sha_path, sha_url, False, verbose)
  38             if os.path.exists(path):
  39                 if verify(path, sha_path, False):
  40                     if verbose:
  41                         print("using already-download file", path)
  42                     return
  43                 else:
  44                     if verbose:
  45                         print("ignoring already-download file",
  46                             path, "due to failed verification")
  47                     os.unlink(path)
  48         download(temp_path, url, True, verbose)
  49         if do_verify and not verify(temp_path, sha_path, verbose):
  50             raise RuntimeError("failed verification")
  51         if verbose:
  52             print("moving {} to {}".format(temp_path, path))
  53         shutil.move(temp_path, path)
  54     finally:
  55         delete_if_present(sha_path, verbose)
  56         delete_if_present(temp_path, verbose)
  57
  58
  59 def delete_if_present(path, verbose):
  60     """Remove the given file if present"""
  61     if os.path.isfile(path):
  62         if verbose:
  63             print("removing", path)
  64         os.unlink(path)
  65
  66
  67 def download(path, url, probably_big, verbose):
  68     for _ in range(0, 4):
  69         try:
  70             _download(path, url, probably_big, verbose, True)
  71             return
  72         except RuntimeError:
  73             print("\nspurious failure, trying again")
  74     _download(path, url, probably_big, verbose, False)
  75
  76
  77 def _download(path, url, probably_big, verbose, exception):
  78     if probably_big or verbose:
  79         print("downloading {}".format(url))
  80     # see http://serverfault.com/questions/301128/how-to-download
  81     if sys.platform == 'win32':
  82         run(["PowerShell.exe", "/nologo", "-Command",
  83              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  84              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  85             verbose=verbose,
  86             exception=exception)
  87     else:
  88         if probably_big or verbose:
  89             option = "-#"
  90         else:
  91             option = "-s"
  92         require(["curl", "--version"])
  93         run(["curl", option,
  94              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  95              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  96              "--retry", "3", "-Sf", "-o", path, url],
  97             verbose=verbose,
  98             exception=exception)
  99
 100
 101 def verify(path, sha_path, verbose):
 102     """Check if the sha256 sum of the given path is valid"""
 103     if verbose:
 104         print("verifying", path)
 105     with open(path, "rb") as source:
 106         found = hashlib.sha256(source.read()).hexdigest()
 107     with open(sha_path, "r") as sha256sum:
 108         expected = sha256sum.readline().split()[0]
 109     verified = found == expected
 110     if not verified:
 111         print("invalid checksum:\n"
 112               "    found:    {}\n"
 113               "    expected: {}".format(found, expected))
 114     return verified
 115
 116
 117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 118     """Unpack the given tarball file"""
 119     print("extracting", tarball)
 120     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 121     with contextlib.closing(tarfile.open(tarball)) as tar:
 122         for member in tar.getnames():
 123             if "/" not in member:
 124                 continue
 125             name = member.replace(fname + "/", "", 1)
 126             if match is not None and not name.startswith(match):
 127                 continue
 128             name = name[len(match) + 1:]
 129
 130             dst_path = os.path.join(dst, name)
 131             if verbose:
 132                 print("  extracting", member)
 133             tar.extract(member, dst)
 134             src_path = os.path.join(dst, member)
 135             if os.path.isdir(src_path) and os.path.exists(dst_path):
 136                 continue
 137             shutil.move(src_path, dst_path)
 138     shutil.rmtree(os.path.join(dst, fname))
 139
 140
 141 def run(args, verbose=False, exception=False, **kwargs):
 142     """Run a child program in a new process"""
 143     if verbose:
 144         print("running: " + ' '.join(args))
 145     sys.stdout.flush()
 146     # Use Popen here instead of call() as it apparently allows powershell on
 147     # Windows to not lock up waiting for input presumably.
 148     ret = subprocess.Popen(args, **kwargs)
 149     code = ret.wait()
 150     if code != 0:
 151         err = "failed to run: " + ' '.join(args)
 152         if verbose or exception:
 153             raise RuntimeError(err)
 154         sys.exit(err)
 155
 156
 157 def require(cmd, exit=True):
 158     '''Run a command, returning its output.
 159     On error,
 160         If `exit` is `True`, exit the process.
 161         Otherwise, return None.'''
 162     try:
 163         return subprocess.check_output(cmd).strip()
 164     except (subprocess.CalledProcessError, OSError) as exc:
 165         if not exit:
 166             return None
 167         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 168         print("Please make sure it's installed and in the path.")
 169         sys.exit(1)
 170
 171
 172 def stage0_data(rust_root):
 173     """Build a dictionary from stage0.txt"""
 174     nightlies = os.path.join(rust_root, "src/stage0.txt")
 175     with open(nightlies, 'r') as nightlies:
 176         lines = [line.rstrip() for line in nightlies
 177                  if not line.startswith("#")]
 178         return dict([line.split(": ", 1) for line in lines if line])
 179
 180
 181 def format_build_time(duration):
 182     """Return a nicer format for build time
 183
 184     >>> format_build_time('300')
 185     '0:05:00'
 186     """
 187     return str(datetime.timedelta(seconds=int(duration)))
 188
 189
 190 def default_build_triple():
 191     """Build triple as in LLVM"""
 192     default_encoding = sys.getdefaultencoding()
 193     required = sys.platform != 'win32'
 194     ostype = require(["uname", "-s"], exit=required)
 195     cputype = require(['uname', '-m'], exit=required)
 196
 197     # If we do not have `uname`, assume Windows.
 198     if ostype is None or cputype is None:
 199         return 'x86_64-pc-windows-msvc'
 200
 201     ostype = ostype.decode(default_encoding)
 202     cputype = cputype.decode(default_encoding)
 203
 204     # The goal here is to come up with the same triple as LLVM would,
 205     # at least for the subset of platforms we're willing to target.
 206     ostype_mapper = {
 207         'Darwin': 'apple-darwin',
 208         'DragonFly': 'unknown-dragonfly',
 209         'FreeBSD': 'unknown-freebsd',
 210         'Haiku': 'unknown-haiku',
 211         'NetBSD': 'unknown-netbsd',
 212         'OpenBSD': 'unknown-openbsd'
 213     }
 214
 215     # Consider the direct transformation first and then the special cases
 216     if ostype in ostype_mapper:
 217         ostype = ostype_mapper[ostype]
 218     elif ostype == 'Linux':
 219         os_from_sp = subprocess.check_output(
 220             ['uname', '-o']).strip().decode(default_encoding)
 221         if os_from_sp == 'Android':
 222             ostype = 'linux-android'
 223         else:
 224             ostype = 'unknown-linux-gnu'
 225     elif ostype == 'SunOS':
 226         ostype = 'sun-solaris'
 227         # On Solaris, uname -m will return a machine classification instead
 228         # of a cpu type, so uname -p is recommended instead.  However, the
 229         # output from that option is too generic for our purposes (it will
 230         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 231         # must be used instead.
 232         cputype = require(['isainfo', '-k']).decode(default_encoding)
 233     elif ostype.startswith('MINGW'):
 234         # msys' `uname` does not print gcc configuration, but prints msys
 235         # configuration. so we cannot believe `uname -m`:
 236         # msys1 is always i686 and msys2 is always x86_64.
 237         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 238         # MINGW64 on x86_64.
 239         ostype = 'pc-windows-gnu'
 240         cputype = 'i686'
 241         if os.environ.get('MSYSTEM') == 'MINGW64':
 242             cputype = 'x86_64'
 243     elif ostype.startswith('MSYS'):
 244         ostype = 'pc-windows-gnu'
 245     elif ostype.startswith('CYGWIN_NT'):
 246         cputype = 'i686'
 247         if ostype.endswith('WOW64'):
 248             cputype = 'x86_64'
 249         ostype = 'pc-windows-gnu'
 250     elif sys.platform == 'win32':
 251         # Some Windows platforms might have a `uname` command that returns a
 252         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 253         # these cases, fall back to using sys.platform.
 254         return 'x86_64-pc-windows-msvc'
 255     else:
 256         err = "unknown OS type: {}".format(ostype)
 257         sys.exit(err)
 258
 259     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 260         cputype = subprocess.check_output(
 261               ['uname', '-p']).strip().decode(default_encoding)
 262     cputype_mapper = {
 263         'BePC': 'i686',
 264         'aarch64': 'aarch64',
 265         'amd64': 'x86_64',
 266         'arm64': 'aarch64',
 267         'i386': 'i686',
 268         'i486': 'i686',
 269         'i686': 'i686',
 270         'i786': 'i686',
 271         'powerpc': 'powerpc',
 272         'powerpc64': 'powerpc64',
 273         'powerpc64le': 'powerpc64le',
 274         'ppc': 'powerpc',
 275         'ppc64': 'powerpc64',
 276         'ppc64le': 'powerpc64le',
 277         's390x': 's390x',
 278         'x64': 'x86_64',
 279         'x86': 'i686',
 280         'x86-64': 'x86_64',
 281         'x86_64': 'x86_64'
 282     }
 283
 284     # Consider the direct transformation first and then the special cases
 285     if cputype in cputype_mapper:
 286         cputype = cputype_mapper[cputype]
 287     elif cputype in {'xscale', 'arm'}:
 288         cputype = 'arm'
 289         if ostype == 'linux-android':
 290             ostype = 'linux-androideabi'
 291         elif ostype == 'unknown-freebsd':
 292             cputype = subprocess.check_output(
 293                 ['uname', '-p']).strip().decode(default_encoding)
 294             ostype = 'unknown-freebsd'
 295     elif cputype == 'armv6l':
 296         cputype = 'arm'
 297         if ostype == 'linux-android':
 298             ostype = 'linux-androideabi'
 299         else:
 300             ostype += 'eabihf'
 301     elif cputype in {'armv7l', 'armv8l'}:
 302         cputype = 'armv7'
 303         if ostype == 'linux-android':
 304             ostype = 'linux-androideabi'
 305         else:
 306             ostype += 'eabihf'
 307     elif cputype == 'mips':
 308         if sys.byteorder == 'big':
 309             cputype = 'mips'
 310         elif sys.byteorder == 'little':
 311             cputype = 'mipsel'
 312         else:
 313             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 314     elif cputype == 'mips64':
 315         if sys.byteorder == 'big':
 316             cputype = 'mips64'
 317         elif sys.byteorder == 'little':
 318             cputype = 'mips64el'
 319         else:
 320             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 321         # only the n64 ABI is supported, indicate it
 322         ostype += 'abi64'
 323     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 324         pass
 325     else:
 326         err = "unknown cpu type: {}".format(cputype)
 327         sys.exit(err)
 328
 329     return "{}-{}".format(cputype, ostype)
 330
 331
 332 @contextlib.contextmanager
 333 def output(filepath):
 334     tmp = filepath + '.tmp'
 335     with open(tmp, 'w') as f:
 336         yield f
 337     try:
 338         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 339         os.rename(tmp, filepath)
 340     except OSError:
 341         shutil.copy2(tmp, filepath)
 342         os.remove(tmp)
 343
 344
 345 class RustBuild(object):
 346     """Provide all the methods required to build Rust"""
 347     def __init__(self):
 348         self.cargo_channel = ''
 349         self.date = ''
 350         self._download_url = ''
 351         self.rustc_channel = ''
 352         self.rustfmt_channel = ''
 353         self.build = ''
 354         self.build_dir = ''
 355         self.clean = False
 356         self.config_toml = ''
 357         self.rust_root = ''
 358         self.use_locked_deps = ''
 359         self.use_vendored_sources = ''
 360         self.verbose = False
 361         self.git_version = None
 362         self.nix_deps_dir = None
 363
 364     def download_stage0(self):
 365         """Fetch the build system for Rust, written in Rust
 366
 367         This method will build a cache directory, then it will fetch the
 368         tarball which has the stage0 compiler used to then bootstrap the Rust
 369         compiler itself.
 370
 371         Each downloaded tarball is extracted, after that, the script
 372         will move all the content to the right place.
 373         """
 374         rustc_channel = self.rustc_channel
 375         cargo_channel = self.cargo_channel
 376         rustfmt_channel = self.rustfmt_channel
 377
 378         if self.rustc().startswith(self.bin_root()) and \
 379                 (not os.path.exists(self.rustc()) or
 380                  self.program_out_of_date(self.rustc_stamp())):
 381             if os.path.exists(self.bin_root()):
 382                 shutil.rmtree(self.bin_root())
 383             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 384             filename = "rust-std-{}-{}{}".format(
 385                 rustc_channel, self.build, tarball_suffix)
 386             pattern = "rust-std-{}".format(self.build)
 387             self._download_stage0_helper(filename, pattern, tarball_suffix)
 388
 389             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 392             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 393             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 394             lib_dir = "{}/lib".format(self.bin_root())
 395             for lib in os.listdir(lib_dir):
 396                 if lib.endswith(".so"):
 397                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 398             with output(self.rustc_stamp()) as rust_stamp:
 399                 rust_stamp.write(self.date)
 400
 401         if self.cargo().startswith(self.bin_root()) and \
 402                 (not os.path.exists(self.cargo()) or
 403                  self.program_out_of_date(self.cargo_stamp())):
 404             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 405             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 406                                               tarball_suffix)
 407             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 408             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 409             with output(self.cargo_stamp()) as cargo_stamp:
 410                 cargo_stamp.write(self.date)
 411
 412         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 413             not os.path.exists(self.rustfmt())
 414             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 415         ):
 416             if rustfmt_channel:
 417                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 418                 [channel, date] = rustfmt_channel.split('-', 1)
 419                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 420                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 421                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 422                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 423                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 424                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 425
 426         if self.downloading_llvm():
 427             # We want the most recent LLVM submodule update to avoid downloading
 428             # LLVM more often than necessary.
 429             #
 430             # This git command finds that commit SHA, looking for bors-authored
 431             # merges that modified src/llvm-project.
 432             #
 433             # This works even in a repository that has not yet initialized
 434             # submodules.
 435             llvm_sha = subprocess.check_output([
 436                 "git", "log", "--author=bors", "--format=%H", "-n1",
 437                 "-m", "--first-parent",
 438                 "--",
 439                 "src/llvm-project",
 440                 "src/bootstrap/download-ci-llvm-stamp",
 441             ]).decode(sys.getdefaultencoding()).strip()
 442             llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
 443             if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
 444                 self._download_ci_llvm(llvm_sha, llvm_assertions)
 445                 for binary in ["llvm-config", "FileCheck"]:
 446                     self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
 447                 with output(self.llvm_stamp()) as llvm_stamp:
 448                     llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
 449
 450     def downloading_llvm(self):
 451         opt = self.get_toml('download-ci-llvm', 'llvm')
 452         return opt == "true" \
 453             or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
 454
 455     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 456         if date is None:
 457             date = self.date
 458         cache_dst = os.path.join(self.build_dir, "cache")
 459         rustc_cache = os.path.join(cache_dst, date)
 460         if not os.path.exists(rustc_cache):
 461             os.makedirs(rustc_cache)
 462
 463         url = "{}/dist/{}".format(self._download_url, date)
 464         tarball = os.path.join(rustc_cache, filename)
 465         if not os.path.exists(tarball):
 466             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 467         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 468
 469     def _download_ci_llvm(self, llvm_sha, llvm_assertions):
 470         cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
 471         cache_dst = os.path.join(self.build_dir, "cache")
 472         rustc_cache = os.path.join(cache_dst, cache_prefix)
 473         if not os.path.exists(rustc_cache):
 474             os.makedirs(rustc_cache)
 475
 476         url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
 477         if llvm_assertions:
 478             url = url.replace('rustc-builds', 'rustc-builds-alt')
 479         tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 480         filename = "rust-dev-nightly-" + self.build + tarball_suffix
 481         tarball = os.path.join(rustc_cache, filename)
 482         if not os.path.exists(tarball):
 483             get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
 484         unpack(tarball, tarball_suffix, self.llvm_root(),
 485                 match="rust-dev",
 486                 verbose=self.verbose)
 487
 488     def fix_bin_or_dylib(self, fname):
 489         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 490         or the RPATH section, to fix the dynamic library search path
 491
 492         This method is only required on NixOS and uses the PatchELF utility to
 493         change the interpreter/RPATH of ELF executables.
 494
 495         Please see https://nixos.org/patchelf.html for more information
 496         """
 497         default_encoding = sys.getdefaultencoding()
 498         try:
 499             ostype = subprocess.check_output(
 500                 ['uname', '-s']).strip().decode(default_encoding)
 501         except subprocess.CalledProcessError:
 502             return
 503         except OSError as reason:
 504             if getattr(reason, 'winerror', None) is not None:
 505                 return
 506             raise reason
 507
 508         if ostype != "Linux":
 509             return
 510
 511         if not os.path.exists("/etc/NIXOS"):
 512             return
 513         if os.path.exists("/lib"):
 514             return
 515
 516         # At this point we're pretty sure the user is running NixOS
 517         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 518         print(nix_os_msg, fname)
 519
 520         # Only build `stage0/.nix-deps` once.
 521         nix_deps_dir = self.nix_deps_dir
 522         if not nix_deps_dir:
 523             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 524             if not os.path.exists(nix_deps_dir):
 525                 os.makedirs(nix_deps_dir)
 526
 527             nix_deps = [
 528                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 529                 "stdenv.cc.bintools",
 530
 531                 # Needed as a system dependency of `libLLVM-*.so`.
 532                 "zlib",
 533
 534                 # Needed for patching ELF binaries (see doc comment above).
 535                 "patchelf",
 536             ]
 537
 538             # Run `nix-build` to "build" each dependency (which will likely reuse
 539             # the existing `/nix/store` copy, or at most download a pre-built copy).
 540             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 541             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 542             # ensuring garbage collection will never remove the `/nix/store` path
 543             # (which would break our patched binaries that hardcode those paths).
 544             for dep in nix_deps:
 545                 try:
 546                     subprocess.check_output([
 547                         "nix-build", "<nixpkgs>",
 548                         "-A", dep,
 549                         "-o", "{}/{}".format(nix_deps_dir, dep),
 550                     ])
 551                 except subprocess.CalledProcessError as reason:
 552                     print("warning: failed to call nix-build:", reason)
 553                     return
 554
 555             self.nix_deps_dir = nix_deps_dir
 556
 557         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 558
 559         if fname.endswith(".so"):
 560             # Dynamic library, patch RPATH to point to system dependencies.
 561             dylib_deps = ["zlib"]
 562             rpath_entries = [
 563                 # Relative default, all binary and dynamic libraries we ship
 564                 # appear to have this (even when `../lib` is redundant).
 565                 "$ORIGIN/../lib",
 566             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 567             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 568         else:
 569             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 570             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 571                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 572
 573         try:
 574             subprocess.check_output([patchelf] + patchelf_args + [fname])
 575         except subprocess.CalledProcessError as reason:
 576             print("warning: failed to call patchelf:", reason)
 577             return
 578
 579     def rustc_stamp(self):
 580         """Return the path for .rustc-stamp
 581
 582         >>> rb = RustBuild()
 583         >>> rb.build_dir = "build"
 584         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 585         True
 586         """
 587         return os.path.join(self.bin_root(), '.rustc-stamp')
 588
 589     def cargo_stamp(self):
 590         """Return the path for .cargo-stamp
 591
 592         >>> rb = RustBuild()
 593         >>> rb.build_dir = "build"
 594         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 595         True
 596         """
 597         return os.path.join(self.bin_root(), '.cargo-stamp')
 598
 599     def rustfmt_stamp(self):
 600         """Return the path for .rustfmt-stamp
 601
 602         >>> rb = RustBuild()
 603         >>> rb.build_dir = "build"
 604         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 605         True
 606         """
 607         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 608
 609     def llvm_stamp(self):
 610         """Return the path for .rustfmt-stamp
 611
 612         >>> rb = RustBuild()
 613         >>> rb.build_dir = "build"
 614         >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
 615         True
 616         """
 617         return os.path.join(self.llvm_root(), '.llvm-stamp')
 618
 619
 620     def program_out_of_date(self, stamp_path, extra=""):
 621         """Check if the given program stamp is out of date"""
 622         if not os.path.exists(stamp_path) or self.clean:
 623             return True
 624         with open(stamp_path, 'r') as stamp:
 625             return (self.date + extra) != stamp.read()
 626
 627     def bin_root(self):
 628         """Return the binary root directory
 629
 630         >>> rb = RustBuild()
 631         >>> rb.build_dir = "build"
 632         >>> rb.bin_root() == os.path.join("build", "stage0")
 633         True
 634
 635         When the 'build' property is given should be a nested directory:
 636
 637         >>> rb.build = "devel"
 638         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 639         True
 640         """
 641         return os.path.join(self.build_dir, self.build, "stage0")
 642
 643     def llvm_root(self):
 644         """Return the CI LLVM root directory
 645
 646         >>> rb = RustBuild()
 647         >>> rb.build_dir = "build"
 648         >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
 649         True
 650
 651         When the 'build' property is given should be a nested directory:
 652
 653         >>> rb.build = "devel"
 654         >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
 655         True
 656         """
 657         return os.path.join(self.build_dir, self.build, "ci-llvm")
 658
 659     def get_toml(self, key, section=None):
 660         """Returns the value of the given key in config.toml, otherwise returns None
 661
 662         >>> rb = RustBuild()
 663         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 664         >>> rb.get_toml("key2")
 665         'value2'
 666
 667         If the key does not exists, the result is None:
 668
 669         >>> rb.get_toml("key3") is None
 670         True
 671
 672         Optionally also matches the section the key appears in
 673
 674         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 675         >>> rb.get_toml('key', 'a')
 676         'value1'
 677         >>> rb.get_toml('key', 'b')
 678         'value2'
 679         >>> rb.get_toml('key', 'c') is None
 680         True
 681
 682         >>> rb.config_toml = 'key1 = true'
 683         >>> rb.get_toml("key1")
 684         'true'
 685         """
 686
 687         cur_section = None
 688         for line in self.config_toml.splitlines():
 689             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 690             if section_match is not None:
 691                 cur_section = section_match.group(1)
 692
 693             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 694             if match is not None:
 695                 value = match.group(1)
 696                 if section is None or section == cur_section:
 697                     return self.get_string(value) or value.strip()
 698         return None
 699
 700     def cargo(self):
 701         """Return config path for cargo"""
 702         return self.program_config('cargo')
 703
 704     def rustc(self):
 705         """Return config path for rustc"""
 706         return self.program_config('rustc')
 707
 708     def rustfmt(self):
 709         """Return config path for rustfmt"""
 710         if not self.rustfmt_channel:
 711             return None
 712         return self.program_config('rustfmt')
 713
 714     def program_config(self, program):
 715         """Return config path for the given program
 716
 717         >>> rb = RustBuild()
 718         >>> rb.config_toml = 'rustc = "rustc"\\n'
 719         >>> rb.program_config('rustc')
 720         'rustc'
 721         >>> rb.config_toml = ''
 722         >>> cargo_path = rb.program_config('cargo')
 723         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 724         ... "bin", "cargo")
 725         True
 726         """
 727         config = self.get_toml(program)
 728         if config:
 729             return os.path.expanduser(config)
 730         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 731             program, self.exe_suffix()))
 732
 733     @staticmethod
 734     def get_string(line):
 735         """Return the value between double quotes
 736
 737         >>> RustBuild.get_string('    "devel"   ')
 738         'devel'
 739         >>> RustBuild.get_string("    'devel'   ")
 740         'devel'
 741         >>> RustBuild.get_string('devel') is None
 742         True
 743         >>> RustBuild.get_string('    "devel   ')
 744         ''
 745         """
 746         start = line.find('"')
 747         if start != -1:
 748             end = start + 1 + line[start + 1:].find('"')
 749             return line[start + 1:end]
 750         start = line.find('\'')
 751         if start != -1:
 752             end = start + 1 + line[start + 1:].find('\'')
 753             return line[start + 1:end]
 754         return None
 755
 756     @staticmethod
 757     def exe_suffix():
 758         """Return a suffix for executables"""
 759         if sys.platform == 'win32':
 760             return '.exe'
 761         return ''
 762
 763     def bootstrap_binary(self):
 764         """Return the path of the bootstrap binary
 765
 766         >>> rb = RustBuild()
 767         >>> rb.build_dir = "build"
 768         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 769         ... "debug", "bootstrap")
 770         True
 771         """
 772         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 773
 774     def build_bootstrap(self):
 775         """Build bootstrap"""
 776         build_dir = os.path.join(self.build_dir, "bootstrap")
 777         if self.clean and os.path.exists(build_dir):
 778             shutil.rmtree(build_dir)
 779         env = os.environ.copy()
 780         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 781         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 782         if "CARGO_BUILD_TARGET" in env:
 783             del env["CARGO_BUILD_TARGET"]
 784         env["CARGO_TARGET_DIR"] = build_dir
 785         env["RUSTC"] = self.rustc()
 786         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 787             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 788             if "LD_LIBRARY_PATH" in env else ""
 789         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 790             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 791             if "DYLD_LIBRARY_PATH" in env else ""
 792         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 793             (os.pathsep + env["LIBRARY_PATH"]) \
 794             if "LIBRARY_PATH" in env else ""
 795         # preserve existing RUSTFLAGS
 796         env.setdefault("RUSTFLAGS", "")
 797         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 798
 799         build_section = "target.{}".format(self.build_triple())
 800         target_features = []
 801         if self.get_toml("crt-static", build_section) == "true":
 802             target_features += ["+crt-static"]
 803         elif self.get_toml("crt-static", build_section) == "false":
 804             target_features += ["-crt-static"]
 805         if target_features:
 806             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 807         target_linker = self.get_toml("linker", build_section)
 808         if target_linker is not None:
 809             env["RUSTFLAGS"] += " -C linker=" + target_linker
 810         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 811         if self.get_toml("deny-warnings", "rust") != "false":
 812             env["RUSTFLAGS"] += " -Dwarnings"
 813
 814         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 815             os.pathsep + env["PATH"]
 816         if not os.path.isfile(self.cargo()):
 817             raise Exception("no cargo executable found at `{}`".format(
 818                 self.cargo()))
 819         args = [self.cargo(), "build", "--manifest-path",
 820                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 821         for _ in range(1, self.verbose):
 822             args.append("--verbose")
 823         if self.use_locked_deps:
 824             args.append("--locked")
 825         if self.use_vendored_sources:
 826             args.append("--frozen")
 827         run(args, env=env, verbose=self.verbose)
 828
 829     def build_triple(self):
 830         """Build triple as in LLVM"""
 831         config = self.get_toml('build')
 832         if config:
 833             return config
 834         return default_build_triple()
 835
 836     def check_submodule(self, module, slow_submodules):
 837         if not slow_submodules:
 838             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 839                                            cwd=os.path.join(self.rust_root, module),
 840                                            stdout=subprocess.PIPE)
 841             return checked_out
 842         else:
 843             return None
 844
 845     def update_submodule(self, module, checked_out, recorded_submodules):
 846         module_path = os.path.join(self.rust_root, module)
 847
 848         if checked_out is not None:
 849             default_encoding = sys.getdefaultencoding()
 850             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 851             if recorded_submodules[module] == checked_out:
 852                 return
 853
 854         print("Updating submodule", module)
 855
 856         run(["git", "submodule", "-q", "sync", module],
 857             cwd=self.rust_root, verbose=self.verbose)
 858
 859         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 860         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 861             update_args.append("--progress")
 862         update_args.append(module)
 863         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 864
 865         run(["git", "reset", "-q", "--hard"],
 866             cwd=module_path, verbose=self.verbose)
 867         run(["git", "clean", "-qdfx"],
 868             cwd=module_path, verbose=self.verbose)
 869
 870     def update_submodules(self):
 871         """Update submodules"""
 872         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 873                 self.get_toml('submodules') == "false":
 874             return
 875
 876         default_encoding = sys.getdefaultencoding()
 877
 878         # check the existence and version of 'git' command
 879         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 880         self.git_version = distutils.version.LooseVersion(git_version_str)
 881
 882         slow_submodules = self.get_toml('fast-submodules') == "false"
 883         start_time = time()
 884         if slow_submodules:
 885             print('Unconditionally updating all submodules')
 886         else:
 887             print('Updating only changed submodules')
 888         default_encoding = sys.getdefaultencoding()
 889         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 890             ["git", "config", "--file",
 891              os.path.join(self.rust_root, ".gitmodules"),
 892              "--get-regexp", "path"]
 893         ).decode(default_encoding).splitlines()]
 894         filtered_submodules = []
 895         submodules_names = []
 896         for module in submodules:
 897             if module.endswith("llvm-project"):
 898                 if self.get_toml('llvm-config') or self.downloading_llvm():
 899                     if self.get_toml('lld') != 'true':
 900                         continue
 901             check = self.check_submodule(module, slow_submodules)
 902             filtered_submodules.append((module, check))
 903             submodules_names.append(module)
 904         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 905                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 906         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 907         recorded_submodules = {}
 908         for data in recorded:
 909             data = data.split()
 910             recorded_submodules[data[3]] = data[2]
 911         for module in filtered_submodules:
 912             self.update_submodule(module[0], module[1], recorded_submodules)
 913         print("Submodules updated in %.2f seconds" % (time() - start_time))
 914
 915     def set_normal_environment(self):
 916         """Set download URL for normal environment"""
 917         if 'RUSTUP_DIST_SERVER' in os.environ:
 918             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 919         else:
 920             self._download_url = 'https://static.rust-lang.org'
 921
 922     def set_dev_environment(self):
 923         """Set download URL for development environment"""
 924         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 925             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 926         else:
 927             self._download_url = 'https://dev-static.rust-lang.org'
 928
 929     def check_vendored_status(self):
 930         """Check that vendoring is configured properly"""
 931         vendor_dir = os.path.join(self.rust_root, 'vendor')
 932         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 933             if os.environ.get('USER') != os.environ['SUDO_USER']:
 934                 self.use_vendored_sources = True
 935                 print('info: looks like you are running this command under `sudo`')
 936                 print('      and so in order to preserve your $HOME this will now')
 937                 print('      use vendored sources by default.')
 938                 if not os.path.exists(vendor_dir):
 939                     print('error: vendoring required, but vendor directory does not exist.')
 940                     print('       Run `cargo vendor` without sudo to initialize the '
 941                           'vendor directory.')
 942                     raise Exception("{} not found".format(vendor_dir))
 943
 944         if self.use_vendored_sources:
 945             if not os.path.exists('.cargo'):
 946                 os.makedirs('.cargo')
 947             with output('.cargo/config') as cargo_config:
 948                 cargo_config.write(
 949                     "[source.crates-io]\n"
 950                     "replace-with = 'vendored-sources'\n"
 951                     "registry = 'https://example.com'\n"
 952                     "\n"
 953                     "[source.vendored-sources]\n"
 954                     "directory = '{}/vendor'\n"
 955                     .format(self.rust_root))
 956         else:
 957             if os.path.exists('.cargo'):
 958                 shutil.rmtree('.cargo')
 959
 960     def ensure_vendored(self):
 961         """Ensure that the vendored sources are available if needed"""
 962         vendor_dir = os.path.join(self.rust_root, 'vendor')
 963         # Note that this does not handle updating the vendored dependencies if
 964         # the rust git repository is updated. Normal development usually does
 965         # not use vendoring, so hopefully this isn't too much of a problem.
 966         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 967             run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
 968                 verbose=self.verbose, cwd=self.rust_root)
 969
 970
 971 def bootstrap(help_triggered):
 972     """Configure, fetch, build and run the initial bootstrap"""
 973
 974     # If the user is asking for help, let them know that the whole download-and-build
 975     # process has to happen before anything is printed out.
 976     if help_triggered:
 977         print("info: Downloading and building bootstrap before processing --help")
 978         print("      command. See src/bootstrap/README.md for help with common")
 979         print("      commands.")
 980
 981     parser = argparse.ArgumentParser(description='Build rust')
 982     parser.add_argument('--config')
 983     parser.add_argument('--build')
 984     parser.add_argument('--clean', action='store_true')
 985     parser.add_argument('-v', '--verbose', action='count', default=0)
 986
 987     args = [a for a in sys.argv if a != '-h' and a != '--help']
 988     args, _ = parser.parse_known_args(args)
 989
 990     # Configure initial bootstrap
 991     build = RustBuild()
 992     build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 993     build.verbose = args.verbose
 994     build.clean = args.clean
 995
 996     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
 997     # exists).
 998     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
 999     if not toml_path and os.path.exists('config.toml'):
1000         toml_path = 'config.toml'
1001
1002     if toml_path:
1003         if not os.path.exists(toml_path):
1004             toml_path = os.path.join(build.rust_root, toml_path)
1005
1006         with open(toml_path) as config:
1007             build.config_toml = config.read()
1008
1009     profile = build.get_toml('profile')
1010     if profile is not None:
1011         include_file = 'config.{}.toml'.format(profile)
1012         include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1013         include_path = os.path.join(include_dir, include_file)
1014         # HACK: This works because `build.get_toml()` returns the first match it finds for a
1015         # specific key, so appending our defaults at the end allows the user to override them
1016         with open(include_path) as included_toml:
1017             build.config_toml += os.linesep + included_toml.read()
1018
1019     config_verbose = build.get_toml('verbose', 'build')
1020     if config_verbose is not None:
1021         build.verbose = max(build.verbose, int(config_verbose))
1022
1023     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1024
1025     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1026
1027     build.check_vendored_status()
1028
1029     build_dir = build.get_toml('build-dir', 'build') or 'build'
1030     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1031
1032     data = stage0_data(build.rust_root)
1033     build.date = data['date']
1034     build.rustc_channel = data['rustc']
1035     build.cargo_channel = data['cargo']
1036
1037     if "rustfmt" in data:
1038         build.rustfmt_channel = data['rustfmt']
1039
1040     if 'dev' in data:
1041         build.set_dev_environment()
1042     else:
1043         build.set_normal_environment()
1044
1045     build.update_submodules()
1046
1047     # Fetch/build the bootstrap
1048     build.build = args.build or build.build_triple()
1049     build.download_stage0()
1050     sys.stdout.flush()
1051     build.ensure_vendored()
1052     build.build_bootstrap()
1053     sys.stdout.flush()
1054
1055     # Run the bootstrap
1056     args = [build.bootstrap_binary()]
1057     args.extend(sys.argv[1:])
1058     env = os.environ.copy()
1059     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1060     env["BOOTSTRAP_PYTHON"] = sys.executable
1061     env["BUILD_DIR"] = build.build_dir
1062     env["RUSTC_BOOTSTRAP"] = '1'
1063     if toml_path:
1064         env["BOOTSTRAP_CONFIG"] = toml_path
1065     run(args, env=env, verbose=build.verbose)
1066
1067
1068 def main():
1069     """Entry point for the bootstrap process"""
1070     start_time = time()
1071
1072     # x.py help <cmd> ...
1073     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1074         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1075
1076     help_triggered = (
1077         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1078     try:
1079         bootstrap(help_triggered)
1080         if not help_triggered:
1081             print("Build completed successfully in {}".format(
1082                 format_build_time(time() - start_time)))
1083     except (SystemExit, KeyboardInterrupt) as error:
1084         if hasattr(error, 'code') and isinstance(error.code, int):
1085             exit_code = error.code
1086         else:
1087             exit_code = 1
1088             print(error)
1089         if not help_triggered:
1090             print("Build completed unsuccessfully in {}".format(
1091                 format_build_time(time() - start_time)))
1092         sys.exit(exit_code)
1093
1094
1095 if __name__ == '__main__':
1096     main()