src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17
  18 def get(url, path, verbose=False):
  19     suffix = '.sha256'
  20     sha_url = url + suffix
  21     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  22         temp_path = temp_file.name
  23     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  24         sha_path = sha_file.name
  25
  26     try:
  27         download(sha_path, sha_url, False, verbose)
  28         if os.path.exists(path):
  29             if verify(path, sha_path, False):
  30                 if verbose:
  31                     print("using already-download file", path)
  32                 return
  33             else:
  34                 if verbose:
  35                     print("ignoring already-download file",
  36                           path, "due to failed verification")
  37                 os.unlink(path)
  38         download(temp_path, url, True, verbose)
  39         if not verify(temp_path, sha_path, verbose):
  40             raise RuntimeError("failed verification")
  41         if verbose:
  42             print("moving {} to {}".format(temp_path, path))
  43         shutil.move(temp_path, path)
  44     finally:
  45         delete_if_present(sha_path, verbose)
  46         delete_if_present(temp_path, verbose)
  47
  48
  49 def delete_if_present(path, verbose):
  50     """Remove the given file if present"""
  51     if os.path.isfile(path):
  52         if verbose:
  53             print("removing", path)
  54         os.unlink(path)
  55
  56
  57 def download(path, url, probably_big, verbose):
  58     for _ in range(0, 4):
  59         try:
  60             _download(path, url, probably_big, verbose, True)
  61             return
  62         except RuntimeError:
  63             print("\nspurious failure, trying again")
  64     _download(path, url, probably_big, verbose, False)
  65
  66
  67 def _download(path, url, probably_big, verbose, exception):
  68     if probably_big or verbose:
  69         print("downloading {}".format(url))
  70     # see http://serverfault.com/questions/301128/how-to-download
  71     if sys.platform == 'win32':
  72         run(["PowerShell.exe", "/nologo", "-Command",
  73              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  74              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  75             verbose=verbose,
  76             exception=exception)
  77     else:
  78         if probably_big or verbose:
  79             option = "-#"
  80         else:
  81             option = "-s"
  82         require(["curl", "--version"])
  83         run(["curl", option,
  84              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  85              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  86              "--retry", "3", "-Sf", "-o", path, url],
  87             verbose=verbose,
  88             exception=exception)
  89
  90
  91 def verify(path, sha_path, verbose):
  92     """Check if the sha256 sum of the given path is valid"""
  93     if verbose:
  94         print("verifying", path)
  95     with open(path, "rb") as source:
  96         found = hashlib.sha256(source.read()).hexdigest()
  97     with open(sha_path, "r") as sha256sum:
  98         expected = sha256sum.readline().split()[0]
  99     verified = found == expected
 100     if not verified:
 101         print("invalid checksum:\n"
 102               "    found:    {}\n"
 103               "    expected: {}".format(found, expected))
 104     return verified
 105
 106
 107 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 108     """Unpack the given tarball file"""
 109     print("extracting", tarball)
 110     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 111     with contextlib.closing(tarfile.open(tarball)) as tar:
 112         for member in tar.getnames():
 113             if "/" not in member:
 114                 continue
 115             name = member.replace(fname + "/", "", 1)
 116             if match is not None and not name.startswith(match):
 117                 continue
 118             name = name[len(match) + 1:]
 119
 120             dst_path = os.path.join(dst, name)
 121             if verbose:
 122                 print("  extracting", member)
 123             tar.extract(member, dst)
 124             src_path = os.path.join(dst, member)
 125             if os.path.isdir(src_path) and os.path.exists(dst_path):
 126                 continue
 127             shutil.move(src_path, dst_path)
 128     shutil.rmtree(os.path.join(dst, fname))
 129
 130
 131 def run(args, verbose=False, exception=False, **kwargs):
 132     """Run a child program in a new process"""
 133     if verbose:
 134         print("running: " + ' '.join(args))
 135     sys.stdout.flush()
 136     # Use Popen here instead of call() as it apparently allows powershell on
 137     # Windows to not lock up waiting for input presumably.
 138     ret = subprocess.Popen(args, **kwargs)
 139     code = ret.wait()
 140     if code != 0:
 141         err = "failed to run: " + ' '.join(args)
 142         if verbose or exception:
 143             raise RuntimeError(err)
 144         sys.exit(err)
 145
 146
 147 def require(cmd, exit=True):
 148     '''Run a command, returning its output.
 149     On error,
 150         If `exit` is `True`, exit the process.
 151         Otherwise, return None.'''
 152     try:
 153         return subprocess.check_output(cmd).strip()
 154     except (subprocess.CalledProcessError, OSError) as exc:
 155         if not exit:
 156             return None
 157         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 158         print("Please make sure it's installed and in the path.")
 159         sys.exit(1)
 160
 161
 162 def stage0_data(rust_root):
 163     """Build a dictionary from stage0.txt"""
 164     nightlies = os.path.join(rust_root, "src/stage0.txt")
 165     with open(nightlies, 'r') as nightlies:
 166         lines = [line.rstrip() for line in nightlies
 167                  if not line.startswith("#")]
 168         return dict([line.split(": ", 1) for line in lines if line])
 169
 170
 171 def format_build_time(duration):
 172     """Return a nicer format for build time
 173
 174     >>> format_build_time('300')
 175     '0:05:00'
 176     """
 177     return str(datetime.timedelta(seconds=int(duration)))
 178
 179
 180 def default_build_triple():
 181     """Build triple as in LLVM"""
 182     default_encoding = sys.getdefaultencoding()
 183     required = sys.platform != 'win32'
 184     ostype = require(["uname", "-s"], exit=required)
 185     cputype = require(['uname', '-m'], exit=required)
 186
 187     # If we do not have `uname`, assume Windows.
 188     if ostype is None or cputype is None:
 189         return 'x86_64-pc-windows-msvc'
 190
 191     ostype = ostype.decode(default_encoding)
 192     cputype = cputype.decode(default_encoding)
 193
 194     # The goal here is to come up with the same triple as LLVM would,
 195     # at least for the subset of platforms we're willing to target.
 196     ostype_mapper = {
 197         'Darwin': 'apple-darwin',
 198         'DragonFly': 'unknown-dragonfly',
 199         'FreeBSD': 'unknown-freebsd',
 200         'Haiku': 'unknown-haiku',
 201         'NetBSD': 'unknown-netbsd',
 202         'OpenBSD': 'unknown-openbsd'
 203     }
 204
 205     # Consider the direct transformation first and then the special cases
 206     if ostype in ostype_mapper:
 207         ostype = ostype_mapper[ostype]
 208     elif ostype == 'Linux':
 209         os_from_sp = subprocess.check_output(
 210             ['uname', '-o']).strip().decode(default_encoding)
 211         if os_from_sp == 'Android':
 212             ostype = 'linux-android'
 213         else:
 214             ostype = 'unknown-linux-gnu'
 215     elif ostype == 'SunOS':
 216         ostype = 'sun-solaris'
 217         # On Solaris, uname -m will return a machine classification instead
 218         # of a cpu type, so uname -p is recommended instead.  However, the
 219         # output from that option is too generic for our purposes (it will
 220         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 221         # must be used instead.
 222         cputype = require(['isainfo', '-k']).decode(default_encoding)
 223     elif ostype.startswith('MINGW'):
 224         # msys' `uname` does not print gcc configuration, but prints msys
 225         # configuration. so we cannot believe `uname -m`:
 226         # msys1 is always i686 and msys2 is always x86_64.
 227         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 228         # MINGW64 on x86_64.
 229         ostype = 'pc-windows-gnu'
 230         cputype = 'i686'
 231         if os.environ.get('MSYSTEM') == 'MINGW64':
 232             cputype = 'x86_64'
 233     elif ostype.startswith('MSYS'):
 234         ostype = 'pc-windows-gnu'
 235     elif ostype.startswith('CYGWIN_NT'):
 236         cputype = 'i686'
 237         if ostype.endswith('WOW64'):
 238             cputype = 'x86_64'
 239         ostype = 'pc-windows-gnu'
 240     elif sys.platform == 'win32':
 241         # Some Windows platforms might have a `uname` command that returns a
 242         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 243         # these cases, fall back to using sys.platform.
 244         return 'x86_64-pc-windows-msvc'
 245     else:
 246         err = "unknown OS type: {}".format(ostype)
 247         sys.exit(err)
 248
 249     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 250         cputype = subprocess.check_output(
 251               ['uname', '-p']).strip().decode(default_encoding)
 252     cputype_mapper = {
 253         'BePC': 'i686',
 254         'aarch64': 'aarch64',
 255         'amd64': 'x86_64',
 256         'arm64': 'aarch64',
 257         'i386': 'i686',
 258         'i486': 'i686',
 259         'i686': 'i686',
 260         'i786': 'i686',
 261         'powerpc': 'powerpc',
 262         'powerpc64': 'powerpc64',
 263         'powerpc64le': 'powerpc64le',
 264         'ppc': 'powerpc',
 265         'ppc64': 'powerpc64',
 266         'ppc64le': 'powerpc64le',
 267         's390x': 's390x',
 268         'x64': 'x86_64',
 269         'x86': 'i686',
 270         'x86-64': 'x86_64',
 271         'x86_64': 'x86_64'
 272     }
 273
 274     # Consider the direct transformation first and then the special cases
 275     if cputype in cputype_mapper:
 276         cputype = cputype_mapper[cputype]
 277     elif cputype in {'xscale', 'arm'}:
 278         cputype = 'arm'
 279         if ostype == 'linux-android':
 280             ostype = 'linux-androideabi'
 281         elif ostype == 'unknown-freebsd':
 282             cputype = subprocess.check_output(
 283                 ['uname', '-p']).strip().decode(default_encoding)
 284             ostype = 'unknown-freebsd'
 285     elif cputype == 'armv6l':
 286         cputype = 'arm'
 287         if ostype == 'linux-android':
 288             ostype = 'linux-androideabi'
 289         else:
 290             ostype += 'eabihf'
 291     elif cputype in {'armv7l', 'armv8l'}:
 292         cputype = 'armv7'
 293         if ostype == 'linux-android':
 294             ostype = 'linux-androideabi'
 295         else:
 296             ostype += 'eabihf'
 297     elif cputype == 'mips':
 298         if sys.byteorder == 'big':
 299             cputype = 'mips'
 300         elif sys.byteorder == 'little':
 301             cputype = 'mipsel'
 302         else:
 303             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 304     elif cputype == 'mips64':
 305         if sys.byteorder == 'big':
 306             cputype = 'mips64'
 307         elif sys.byteorder == 'little':
 308             cputype = 'mips64el'
 309         else:
 310             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 311         # only the n64 ABI is supported, indicate it
 312         ostype += 'abi64'
 313     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 314         pass
 315     else:
 316         err = "unknown cpu type: {}".format(cputype)
 317         sys.exit(err)
 318
 319     return "{}-{}".format(cputype, ostype)
 320
 321
 322 @contextlib.contextmanager
 323 def output(filepath):
 324     tmp = filepath + '.tmp'
 325     with open(tmp, 'w') as f:
 326         yield f
 327     try:
 328         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 329         os.rename(tmp, filepath)
 330     except OSError:
 331         shutil.copy2(tmp, filepath)
 332         os.remove(tmp)
 333
 334
 335 class RustBuild(object):
 336     """Provide all the methods required to build Rust"""
 337     def __init__(self):
 338         self.cargo_channel = ''
 339         self.date = ''
 340         self._download_url = ''
 341         self.rustc_channel = ''
 342         self.rustfmt_channel = ''
 343         self.build = ''
 344         self.build_dir = ''
 345         self.clean = False
 346         self.config_toml = ''
 347         self.rust_root = ''
 348         self.use_locked_deps = ''
 349         self.use_vendored_sources = ''
 350         self.verbose = False
 351         self.git_version = None
 352         self.nix_deps_dir = None
 353
 354     def download_stage0(self):
 355         """Fetch the build system for Rust, written in Rust
 356
 357         This method will build a cache directory, then it will fetch the
 358         tarball which has the stage0 compiler used to then bootstrap the Rust
 359         compiler itself.
 360
 361         Each downloaded tarball is extracted, after that, the script
 362         will move all the content to the right place.
 363         """
 364         rustc_channel = self.rustc_channel
 365         cargo_channel = self.cargo_channel
 366         rustfmt_channel = self.rustfmt_channel
 367
 368         def support_xz():
 369             try:
 370                 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
 371                     temp_path = temp_file.name
 372                 with tarfile.open(temp_path, "w:xz"):
 373                     pass
 374                 return True
 375             except tarfile.CompressionError:
 376                 return False
 377
 378         if self.rustc().startswith(self.bin_root()) and \
 379                 (not os.path.exists(self.rustc()) or
 380                  self.program_out_of_date(self.rustc_stamp())):
 381             if os.path.exists(self.bin_root()):
 382                 shutil.rmtree(self.bin_root())
 383             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 384             filename = "rust-std-{}-{}{}".format(
 385                 rustc_channel, self.build, tarball_suffix)
 386             pattern = "rust-std-{}".format(self.build)
 387             self._download_stage0_helper(filename, pattern, tarball_suffix)
 388
 389             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 392             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 393             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 394             lib_dir = "{}/lib".format(self.bin_root())
 395             for lib in os.listdir(lib_dir):
 396                 if lib.endswith(".so"):
 397                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 398             with output(self.rustc_stamp()) as rust_stamp:
 399                 rust_stamp.write(self.date)
 400
 401             # This is required so that we don't mix incompatible MinGW
 402             # libraries/binaries that are included in rust-std with
 403             # the system MinGW ones.
 404             if "pc-windows-gnu" in self.build:
 405                 filename = "rust-mingw-{}-{}{}".format(
 406                     rustc_channel, self.build, tarball_suffix)
 407                 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
 408
 409         if self.cargo().startswith(self.bin_root()) and \
 410                 (not os.path.exists(self.cargo()) or
 411                  self.program_out_of_date(self.cargo_stamp())):
 412             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 413             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 414                                               tarball_suffix)
 415             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 416             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 417             with output(self.cargo_stamp()) as cargo_stamp:
 418                 cargo_stamp.write(self.date)
 419
 420         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 421             not os.path.exists(self.rustfmt())
 422             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 423         ):
 424             if rustfmt_channel:
 425                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 426                 [channel, date] = rustfmt_channel.split('-', 1)
 427                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 428                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 429                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 430                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 431                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 432                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 433
 434     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 435         if date is None:
 436             date = self.date
 437         cache_dst = os.path.join(self.build_dir, "cache")
 438         rustc_cache = os.path.join(cache_dst, date)
 439         if not os.path.exists(rustc_cache):
 440             os.makedirs(rustc_cache)
 441
 442         url = "{}/dist/{}".format(self._download_url, date)
 443         tarball = os.path.join(rustc_cache, filename)
 444         if not os.path.exists(tarball):
 445             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 446         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 447
 448     def fix_bin_or_dylib(self, fname):
 449         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 450         or the RPATH section, to fix the dynamic library search path
 451
 452         This method is only required on NixOS and uses the PatchELF utility to
 453         change the interpreter/RPATH of ELF executables.
 454
 455         Please see https://nixos.org/patchelf.html for more information
 456         """
 457         default_encoding = sys.getdefaultencoding()
 458         try:
 459             ostype = subprocess.check_output(
 460                 ['uname', '-s']).strip().decode(default_encoding)
 461         except subprocess.CalledProcessError:
 462             return
 463         except OSError as reason:
 464             if getattr(reason, 'winerror', None) is not None:
 465                 return
 466             raise reason
 467
 468         if ostype != "Linux":
 469             return
 470
 471         if not os.path.exists("/etc/NIXOS"):
 472             return
 473         if os.path.exists("/lib"):
 474             return
 475
 476         # At this point we're pretty sure the user is running NixOS
 477         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 478         print(nix_os_msg, fname)
 479
 480         # Only build `stage0/.nix-deps` once.
 481         nix_deps_dir = self.nix_deps_dir
 482         if not nix_deps_dir:
 483             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 484             if not os.path.exists(nix_deps_dir):
 485                 os.makedirs(nix_deps_dir)
 486
 487             nix_deps = [
 488                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 489                 "stdenv.cc.bintools",
 490
 491                 # Needed as a system dependency of `libLLVM-*.so`.
 492                 "zlib",
 493
 494                 # Needed for patching ELF binaries (see doc comment above).
 495                 "patchelf",
 496             ]
 497
 498             # Run `nix-build` to "build" each dependency (which will likely reuse
 499             # the existing `/nix/store` copy, or at most download a pre-built copy).
 500             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 501             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 502             # ensuring garbage collection will never remove the `/nix/store` path
 503             # (which would break our patched binaries that hardcode those paths).
 504             for dep in nix_deps:
 505                 try:
 506                     subprocess.check_output([
 507                         "nix-build", "<nixpkgs>",
 508                         "-A", dep,
 509                         "-o", "{}/{}".format(nix_deps_dir, dep),
 510                     ])
 511                 except subprocess.CalledProcessError as reason:
 512                     print("warning: failed to call nix-build:", reason)
 513                     return
 514
 515             self.nix_deps_dir = nix_deps_dir
 516
 517         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 518
 519         if fname.endswith(".so"):
 520             # Dynamic library, patch RPATH to point to system dependencies.
 521             dylib_deps = ["zlib"]
 522             rpath_entries = [
 523                 # Relative default, all binary and dynamic libraries we ship
 524                 # appear to have this (even when `../lib` is redundant).
 525                 "$ORIGIN/../lib",
 526             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 527             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 528         else:
 529             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 530             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 531                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 532
 533         try:
 534             subprocess.check_output([patchelf] + patchelf_args + [fname])
 535         except subprocess.CalledProcessError as reason:
 536             print("warning: failed to call patchelf:", reason)
 537             return
 538
 539     def rustc_stamp(self):
 540         """Return the path for .rustc-stamp
 541
 542         >>> rb = RustBuild()
 543         >>> rb.build_dir = "build"
 544         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 545         True
 546         """
 547         return os.path.join(self.bin_root(), '.rustc-stamp')
 548
 549     def cargo_stamp(self):
 550         """Return the path for .cargo-stamp
 551
 552         >>> rb = RustBuild()
 553         >>> rb.build_dir = "build"
 554         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 555         True
 556         """
 557         return os.path.join(self.bin_root(), '.cargo-stamp')
 558
 559     def rustfmt_stamp(self):
 560         """Return the path for .rustfmt-stamp
 561
 562         >>> rb = RustBuild()
 563         >>> rb.build_dir = "build"
 564         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 565         True
 566         """
 567         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 568
 569     def program_out_of_date(self, stamp_path, extra=""):
 570         """Check if the given program stamp is out of date"""
 571         if not os.path.exists(stamp_path) or self.clean:
 572             return True
 573         with open(stamp_path, 'r') as stamp:
 574             return (self.date + extra) != stamp.read()
 575
 576     def bin_root(self):
 577         """Return the binary root directory
 578
 579         >>> rb = RustBuild()
 580         >>> rb.build_dir = "build"
 581         >>> rb.bin_root() == os.path.join("build", "stage0")
 582         True
 583
 584         When the 'build' property is given should be a nested directory:
 585
 586         >>> rb.build = "devel"
 587         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 588         True
 589         """
 590         return os.path.join(self.build_dir, self.build, "stage0")
 591
 592     def get_toml(self, key, section=None):
 593         """Returns the value of the given key in config.toml, otherwise returns None
 594
 595         >>> rb = RustBuild()
 596         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 597         >>> rb.get_toml("key2")
 598         'value2'
 599
 600         If the key does not exists, the result is None:
 601
 602         >>> rb.get_toml("key3") is None
 603         True
 604
 605         Optionally also matches the section the key appears in
 606
 607         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 608         >>> rb.get_toml('key', 'a')
 609         'value1'
 610         >>> rb.get_toml('key', 'b')
 611         'value2'
 612         >>> rb.get_toml('key', 'c') is None
 613         True
 614
 615         >>> rb.config_toml = 'key1 = true'
 616         >>> rb.get_toml("key1")
 617         'true'
 618         """
 619
 620         cur_section = None
 621         for line in self.config_toml.splitlines():
 622             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 623             if section_match is not None:
 624                 cur_section = section_match.group(1)
 625
 626             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 627             if match is not None:
 628                 value = match.group(1)
 629                 if section is None or section == cur_section:
 630                     return self.get_string(value) or value.strip()
 631         return None
 632
 633     def cargo(self):
 634         """Return config path for cargo"""
 635         return self.program_config('cargo')
 636
 637     def rustc(self):
 638         """Return config path for rustc"""
 639         return self.program_config('rustc')
 640
 641     def rustfmt(self):
 642         """Return config path for rustfmt"""
 643         if not self.rustfmt_channel:
 644             return None
 645         return self.program_config('rustfmt')
 646
 647     def program_config(self, program):
 648         """Return config path for the given program
 649
 650         >>> rb = RustBuild()
 651         >>> rb.config_toml = 'rustc = "rustc"\\n'
 652         >>> rb.program_config('rustc')
 653         'rustc'
 654         >>> rb.config_toml = ''
 655         >>> cargo_path = rb.program_config('cargo')
 656         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 657         ... "bin", "cargo")
 658         True
 659         """
 660         config = self.get_toml(program)
 661         if config:
 662             return os.path.expanduser(config)
 663         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 664             program, self.exe_suffix()))
 665
 666     @staticmethod
 667     def get_string(line):
 668         """Return the value between double quotes
 669
 670         >>> RustBuild.get_string('    "devel"   ')
 671         'devel'
 672         >>> RustBuild.get_string("    'devel'   ")
 673         'devel'
 674         >>> RustBuild.get_string('devel') is None
 675         True
 676         >>> RustBuild.get_string('    "devel   ')
 677         ''
 678         """
 679         start = line.find('"')
 680         if start != -1:
 681             end = start + 1 + line[start + 1:].find('"')
 682             return line[start + 1:end]
 683         start = line.find('\'')
 684         if start != -1:
 685             end = start + 1 + line[start + 1:].find('\'')
 686             return line[start + 1:end]
 687         return None
 688
 689     @staticmethod
 690     def exe_suffix():
 691         """Return a suffix for executables"""
 692         if sys.platform == 'win32':
 693             return '.exe'
 694         return ''
 695
 696     def bootstrap_binary(self):
 697         """Return the path of the bootstrap binary
 698
 699         >>> rb = RustBuild()
 700         >>> rb.build_dir = "build"
 701         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 702         ... "debug", "bootstrap")
 703         True
 704         """
 705         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 706
 707     def build_bootstrap(self):
 708         """Build bootstrap"""
 709         build_dir = os.path.join(self.build_dir, "bootstrap")
 710         if self.clean and os.path.exists(build_dir):
 711             shutil.rmtree(build_dir)
 712         env = os.environ.copy()
 713         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 714         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 715         if "CARGO_BUILD_TARGET" in env:
 716             del env["CARGO_BUILD_TARGET"]
 717         env["RUSTC_BOOTSTRAP"] = '1'
 718         env["CARGO_TARGET_DIR"] = build_dir
 719         env["RUSTC"] = self.rustc()
 720         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 721             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 722             if "LD_LIBRARY_PATH" in env else ""
 723         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 724             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 725             if "DYLD_LIBRARY_PATH" in env else ""
 726         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 727             (os.pathsep + env["LIBRARY_PATH"]) \
 728             if "LIBRARY_PATH" in env else ""
 729         # preserve existing RUSTFLAGS
 730         env.setdefault("RUSTFLAGS", "")
 731         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 732
 733         build_section = "target.{}".format(self.build_triple())
 734         target_features = []
 735         if self.get_toml("crt-static", build_section) == "true":
 736             target_features += ["+crt-static"]
 737         elif self.get_toml("crt-static", build_section) == "false":
 738             target_features += ["-crt-static"]
 739         if target_features:
 740             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 741         target_linker = self.get_toml("linker", build_section)
 742         if target_linker is not None:
 743             env["RUSTFLAGS"] += " -C linker=" + target_linker
 744         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 745         if self.get_toml("deny-warnings", "rust") != "false":
 746             env["RUSTFLAGS"] += " -Dwarnings"
 747
 748         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 749             os.pathsep + env["PATH"]
 750         if not os.path.isfile(self.cargo()):
 751             raise Exception("no cargo executable found at `{}`".format(
 752                 self.cargo()))
 753         args = [self.cargo(), "build", "--manifest-path",
 754                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 755         for _ in range(1, self.verbose):
 756             args.append("--verbose")
 757         if self.use_locked_deps:
 758             args.append("--locked")
 759         if self.use_vendored_sources:
 760             args.append("--frozen")
 761         run(args, env=env, verbose=self.verbose)
 762
 763     def build_triple(self):
 764         """Build triple as in LLVM"""
 765         config = self.get_toml('build')
 766         if config:
 767             return config
 768         return default_build_triple()
 769
 770     def check_submodule(self, module, slow_submodules):
 771         if not slow_submodules:
 772             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 773                                            cwd=os.path.join(self.rust_root, module),
 774                                            stdout=subprocess.PIPE)
 775             return checked_out
 776         else:
 777             return None
 778
 779     def update_submodule(self, module, checked_out, recorded_submodules):
 780         module_path = os.path.join(self.rust_root, module)
 781
 782         if checked_out is not None:
 783             default_encoding = sys.getdefaultencoding()
 784             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 785             if recorded_submodules[module] == checked_out:
 786                 return
 787
 788         print("Updating submodule", module)
 789
 790         run(["git", "submodule", "-q", "sync", module],
 791             cwd=self.rust_root, verbose=self.verbose)
 792
 793         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 794         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 795             update_args.append("--progress")
 796         update_args.append(module)
 797         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 798
 799         run(["git", "reset", "-q", "--hard"],
 800             cwd=module_path, verbose=self.verbose)
 801         run(["git", "clean", "-qdfx"],
 802             cwd=module_path, verbose=self.verbose)
 803
 804     def update_submodules(self):
 805         """Update submodules"""
 806         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 807                 self.get_toml('submodules') == "false":
 808             return
 809
 810         default_encoding = sys.getdefaultencoding()
 811
 812         # check the existence and version of 'git' command
 813         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 814         self.git_version = distutils.version.LooseVersion(git_version_str)
 815
 816         slow_submodules = self.get_toml('fast-submodules') == "false"
 817         start_time = time()
 818         if slow_submodules:
 819             print('Unconditionally updating all submodules')
 820         else:
 821             print('Updating only changed submodules')
 822         default_encoding = sys.getdefaultencoding()
 823         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 824             ["git", "config", "--file",
 825              os.path.join(self.rust_root, ".gitmodules"),
 826              "--get-regexp", "path"]
 827         ).decode(default_encoding).splitlines()]
 828         filtered_submodules = []
 829         submodules_names = []
 830         for module in submodules:
 831             if module.endswith("llvm-project"):
 832                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 833                     continue
 834             check = self.check_submodule(module, slow_submodules)
 835             filtered_submodules.append((module, check))
 836             submodules_names.append(module)
 837         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 838                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 839         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 840         recorded_submodules = {}
 841         for data in recorded:
 842             data = data.split()
 843             recorded_submodules[data[3]] = data[2]
 844         for module in filtered_submodules:
 845             self.update_submodule(module[0], module[1], recorded_submodules)
 846         print("Submodules updated in %.2f seconds" % (time() - start_time))
 847
 848     def set_normal_environment(self):
 849         """Set download URL for normal environment"""
 850         if 'RUSTUP_DIST_SERVER' in os.environ:
 851             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 852         else:
 853             self._download_url = 'https://static.rust-lang.org'
 854
 855     def set_dev_environment(self):
 856         """Set download URL for development environment"""
 857         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 858             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 859         else:
 860             self._download_url = 'https://dev-static.rust-lang.org'
 861
 862     def check_vendored_status(self):
 863         """Check that vendoring is configured properly"""
 864         vendor_dir = os.path.join(self.rust_root, 'vendor')
 865         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 866             if os.environ.get('USER') != os.environ['SUDO_USER']:
 867                 self.use_vendored_sources = True
 868                 print('info: looks like you are running this command under `sudo`')
 869                 print('      and so in order to preserve your $HOME this will now')
 870                 print('      use vendored sources by default.')
 871                 if not os.path.exists(vendor_dir):
 872                     print('error: vendoring required, but vendor directory does not exist.')
 873                     print('       Run `cargo vendor` without sudo to initialize the '
 874                           'vendor directory.')
 875                     raise Exception("{} not found".format(vendor_dir))
 876
 877         if self.use_vendored_sources:
 878             if not os.path.exists('.cargo'):
 879                 os.makedirs('.cargo')
 880             with output('.cargo/config') as cargo_config:
 881                 cargo_config.write(
 882                     "[source.crates-io]\n"
 883                     "replace-with = 'vendored-sources'\n"
 884                     "registry = 'https://example.com'\n"
 885                     "\n"
 886                     "[source.vendored-sources]\n"
 887                     "directory = '{}/vendor'\n"
 888                     .format(self.rust_root))
 889         else:
 890             if os.path.exists('.cargo'):
 891                 shutil.rmtree('.cargo')
 892
 893     def ensure_vendored(self):
 894         """Ensure that the vendored sources are available if needed"""
 895         vendor_dir = os.path.join(self.rust_root, 'vendor')
 896         # Note that this does not handle updating the vendored dependencies if
 897         # the rust git repository is updated. Normal development usually does
 898         # not use vendoring, so hopefully this isn't too much of a problem.
 899         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 900             run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
 901                 verbose=self.verbose, cwd=self.rust_root)
 902
 903
 904 def bootstrap(help_triggered):
 905     """Configure, fetch, build and run the initial bootstrap"""
 906
 907     # If the user is asking for help, let them know that the whole download-and-build
 908     # process has to happen before anything is printed out.
 909     if help_triggered:
 910         print("info: Downloading and building bootstrap before processing --help")
 911         print("      command. See src/bootstrap/README.md for help with common")
 912         print("      commands.")
 913
 914     parser = argparse.ArgumentParser(description='Build rust')
 915     parser.add_argument('--config')
 916     parser.add_argument('--build')
 917     parser.add_argument('--src')
 918     parser.add_argument('--clean', action='store_true')
 919     parser.add_argument('-v', '--verbose', action='count', default=0)
 920
 921     args = [a for a in sys.argv if a != '-h' and a != '--help']
 922     args, _ = parser.parse_known_args(args)
 923
 924     # Configure initial bootstrap
 925     build = RustBuild()
 926     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 927     build.verbose = args.verbose
 928     build.clean = args.clean
 929
 930     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
 931     # exists).
 932     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
 933     if not toml_path and os.path.exists('config.toml'):
 934         toml_path = 'config.toml'
 935
 936     if toml_path:
 937         if not os.path.exists(toml_path):
 938             toml_path = os.path.join(build.rust_root, toml_path)
 939
 940         with open(toml_path) as config:
 941             build.config_toml = config.read()
 942
 943     config_verbose = build.get_toml('verbose', 'build')
 944     if config_verbose is not None:
 945         build.verbose = max(build.verbose, int(config_verbose))
 946
 947     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 948
 949     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 950
 951     build.check_vendored_status()
 952
 953     build_dir = build.get_toml('build-dir', 'build') or 'build'
 954     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
 955
 956     data = stage0_data(build.rust_root)
 957     build.date = data['date']
 958     build.rustc_channel = data['rustc']
 959     build.cargo_channel = data['cargo']
 960
 961     if "rustfmt" in data:
 962         build.rustfmt_channel = data['rustfmt']
 963
 964     if 'dev' in data:
 965         build.set_dev_environment()
 966     else:
 967         build.set_normal_environment()
 968
 969     build.update_submodules()
 970
 971     # Fetch/build the bootstrap
 972     build.build = args.build or build.build_triple()
 973     build.download_stage0()
 974     sys.stdout.flush()
 975     build.ensure_vendored()
 976     build.build_bootstrap()
 977     sys.stdout.flush()
 978
 979     # Run the bootstrap
 980     args = [build.bootstrap_binary()]
 981     args.extend(sys.argv[1:])
 982     env = os.environ.copy()
 983     env["BUILD"] = build.build
 984     env["SRC"] = build.rust_root
 985     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 986     env["BOOTSTRAP_PYTHON"] = sys.executable
 987     env["BUILD_DIR"] = build.build_dir
 988     env["RUSTC_BOOTSTRAP"] = '1'
 989     env["CARGO"] = build.cargo()
 990     env["RUSTC"] = build.rustc()
 991     if toml_path:
 992         env["BOOTSTRAP_CONFIG"] = toml_path
 993     if build.rustfmt():
 994         env["RUSTFMT"] = build.rustfmt()
 995     run(args, env=env, verbose=build.verbose)
 996
 997
 998 def main():
 999     """Entry point for the bootstrap process"""
1000     start_time = time()
1001
1002     # x.py help <cmd> ...
1003     if len(sys.argv) > 1 and sys.argv[1] == 'help':
1004         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1005
1006     help_triggered = (
1007         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1008     try:
1009         bootstrap(help_triggered)
1010         if not help_triggered:
1011             print("Build completed successfully in {}".format(
1012                 format_build_time(time() - start_time)))
1013     except (SystemExit, KeyboardInterrupt) as error:
1014         if hasattr(error, 'code') and isinstance(error.code, int):
1015             exit_code = error.code
1016         else:
1017             exit_code = 1
1018             print(error)
1019         if not help_triggered:
1020             print("Build completed unsuccessfully in {}".format(
1021                 format_build_time(time() - start_time)))
1022         sys.exit(exit_code)
1023
1024
1025 if __name__ == '__main__':
1026     main()