src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import distutils.version
   6 import hashlib
   7 import os
   8 import re
   9 import shutil
  10 import subprocess
  11 import sys
  12 import tarfile
  13 import tempfile
  14
  15 from time import time
  16
  17
  18 def get(url, path, verbose=False):
  19     suffix = '.sha256'
  20     sha_url = url + suffix
  21     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  22         temp_path = temp_file.name
  23     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  24         sha_path = sha_file.name
  25
  26     try:
  27         download(sha_path, sha_url, False, verbose)
  28         if os.path.exists(path):
  29             if verify(path, sha_path, False):
  30                 if verbose:
  31                     print("using already-download file", path)
  32                 return
  33             else:
  34                 if verbose:
  35                     print("ignoring already-download file",
  36                           path, "due to failed verification")
  37                 os.unlink(path)
  38         download(temp_path, url, True, verbose)
  39         if not verify(temp_path, sha_path, verbose):
  40             raise RuntimeError("failed verification")
  41         if verbose:
  42             print("moving {} to {}".format(temp_path, path))
  43         shutil.move(temp_path, path)
  44     finally:
  45         delete_if_present(sha_path, verbose)
  46         delete_if_present(temp_path, verbose)
  47
  48
  49 def delete_if_present(path, verbose):
  50     """Remove the given file if present"""
  51     if os.path.isfile(path):
  52         if verbose:
  53             print("removing", path)
  54         os.unlink(path)
  55
  56
  57 def download(path, url, probably_big, verbose):
  58     for _ in range(0, 4):
  59         try:
  60             _download(path, url, probably_big, verbose, True)
  61             return
  62         except RuntimeError:
  63             print("\nspurious failure, trying again")
  64     _download(path, url, probably_big, verbose, False)
  65
  66
  67 def _download(path, url, probably_big, verbose, exception):
  68     if probably_big or verbose:
  69         print("downloading {}".format(url))
  70     # see http://serverfault.com/questions/301128/how-to-download
  71     if sys.platform == 'win32':
  72         run(["PowerShell.exe", "/nologo", "-Command",
  73              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  74              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  75             verbose=verbose,
  76             exception=exception)
  77     else:
  78         if probably_big or verbose:
  79             option = "-#"
  80         else:
  81             option = "-s"
  82         require(["curl", "--version"])
  83         run(["curl", option,
  84              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  85              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  86              "--retry", "3", "-Sf", "-o", path, url],
  87             verbose=verbose,
  88             exception=exception)
  89
  90
  91 def verify(path, sha_path, verbose):
  92     """Check if the sha256 sum of the given path is valid"""
  93     if verbose:
  94         print("verifying", path)
  95     with open(path, "rb") as source:
  96         found = hashlib.sha256(source.read()).hexdigest()
  97     with open(sha_path, "r") as sha256sum:
  98         expected = sha256sum.readline().split()[0]
  99     verified = found == expected
 100     if not verified:
 101         print("invalid checksum:\n"
 102               "    found:    {}\n"
 103               "    expected: {}".format(found, expected))
 104     return verified
 105
 106
 107 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 108     """Unpack the given tarball file"""
 109     print("extracting", tarball)
 110     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 111     with contextlib.closing(tarfile.open(tarball)) as tar:
 112         for member in tar.getnames():
 113             if "/" not in member:
 114                 continue
 115             name = member.replace(fname + "/", "", 1)
 116             if match is not None and not name.startswith(match):
 117                 continue
 118             name = name[len(match) + 1:]
 119
 120             dst_path = os.path.join(dst, name)
 121             if verbose:
 122                 print("  extracting", member)
 123             tar.extract(member, dst)
 124             src_path = os.path.join(dst, member)
 125             if os.path.isdir(src_path) and os.path.exists(dst_path):
 126                 continue
 127             shutil.move(src_path, dst_path)
 128     shutil.rmtree(os.path.join(dst, fname))
 129
 130
 131 def run(args, verbose=False, exception=False, **kwargs):
 132     """Run a child program in a new process"""
 133     if verbose:
 134         print("running: " + ' '.join(args))
 135     sys.stdout.flush()
 136     # Use Popen here instead of call() as it apparently allows powershell on
 137     # Windows to not lock up waiting for input presumably.
 138     ret = subprocess.Popen(args, **kwargs)
 139     code = ret.wait()
 140     if code != 0:
 141         err = "failed to run: " + ' '.join(args)
 142         if verbose or exception:
 143             raise RuntimeError(err)
 144         sys.exit(err)
 145
 146
 147 def require(cmd, exit=True):
 148     '''Run a command, returning its output.
 149     On error,
 150         If `exit` is `True`, exit the process.
 151         Otherwise, return None.'''
 152     try:
 153         return subprocess.check_output(cmd).strip()
 154     except (subprocess.CalledProcessError, OSError) as exc:
 155         if not exit:
 156             return None
 157         print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
 158         print("Please make sure it's installed and in the path.")
 159         sys.exit(1)
 160
 161
 162 def stage0_data(rust_root):
 163     """Build a dictionary from stage0.txt"""
 164     nightlies = os.path.join(rust_root, "src/stage0.txt")
 165     with open(nightlies, 'r') as nightlies:
 166         lines = [line.rstrip() for line in nightlies
 167                  if not line.startswith("#")]
 168         return dict([line.split(": ", 1) for line in lines if line])
 169
 170
 171 def format_build_time(duration):
 172     """Return a nicer format for build time
 173
 174     >>> format_build_time('300')
 175     '0:05:00'
 176     """
 177     return str(datetime.timedelta(seconds=int(duration)))
 178
 179
 180 def default_build_triple():
 181     """Build triple as in LLVM"""
 182     default_encoding = sys.getdefaultencoding()
 183     required = sys.platform != 'win32'
 184     ostype = require(["uname", "-s"], exit=required)
 185     cputype = require(['uname', '-m'], exit=required)
 186
 187     # If we do not have `uname`, assume Windows.
 188     if ostype is None or cputype is None:
 189         return 'x86_64-pc-windows-msvc'
 190
 191     ostype = ostype.decode(default_encoding)
 192     cputype = cputype.decode(default_encoding)
 193
 194     # The goal here is to come up with the same triple as LLVM would,
 195     # at least for the subset of platforms we're willing to target.
 196     ostype_mapper = {
 197         'Darwin': 'apple-darwin',
 198         'DragonFly': 'unknown-dragonfly',
 199         'FreeBSD': 'unknown-freebsd',
 200         'Haiku': 'unknown-haiku',
 201         'NetBSD': 'unknown-netbsd',
 202         'OpenBSD': 'unknown-openbsd'
 203     }
 204
 205     # Consider the direct transformation first and then the special cases
 206     if ostype in ostype_mapper:
 207         ostype = ostype_mapper[ostype]
 208     elif ostype == 'Linux':
 209         os_from_sp = subprocess.check_output(
 210             ['uname', '-o']).strip().decode(default_encoding)
 211         if os_from_sp == 'Android':
 212             ostype = 'linux-android'
 213         else:
 214             ostype = 'unknown-linux-gnu'
 215     elif ostype == 'SunOS':
 216         ostype = 'sun-solaris'
 217         # On Solaris, uname -m will return a machine classification instead
 218         # of a cpu type, so uname -p is recommended instead.  However, the
 219         # output from that option is too generic for our purposes (it will
 220         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 221         # must be used instead.
 222         cputype = require(['isainfo', '-k']).decode(default_encoding)
 223     elif ostype.startswith('MINGW'):
 224         # msys' `uname` does not print gcc configuration, but prints msys
 225         # configuration. so we cannot believe `uname -m`:
 226         # msys1 is always i686 and msys2 is always x86_64.
 227         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 228         # MINGW64 on x86_64.
 229         ostype = 'pc-windows-gnu'
 230         cputype = 'i686'
 231         if os.environ.get('MSYSTEM') == 'MINGW64':
 232             cputype = 'x86_64'
 233     elif ostype.startswith('MSYS'):
 234         ostype = 'pc-windows-gnu'
 235     elif ostype.startswith('CYGWIN_NT'):
 236         cputype = 'i686'
 237         if ostype.endswith('WOW64'):
 238             cputype = 'x86_64'
 239         ostype = 'pc-windows-gnu'
 240     elif sys.platform == 'win32':
 241         # Some Windows platforms might have a `uname` command that returns a
 242         # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
 243         # these cases, fall back to using sys.platform.
 244         return 'x86_64-pc-windows-msvc'
 245     else:
 246         err = "unknown OS type: {}".format(ostype)
 247         sys.exit(err)
 248
 249     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 250         cputype = subprocess.check_output(
 251               ['uname', '-p']).strip().decode(default_encoding)
 252     cputype_mapper = {
 253         'BePC': 'i686',
 254         'aarch64': 'aarch64',
 255         'amd64': 'x86_64',
 256         'arm64': 'aarch64',
 257         'i386': 'i686',
 258         'i486': 'i686',
 259         'i686': 'i686',
 260         'i786': 'i686',
 261         'powerpc': 'powerpc',
 262         'powerpc64': 'powerpc64',
 263         'powerpc64le': 'powerpc64le',
 264         'ppc': 'powerpc',
 265         'ppc64': 'powerpc64',
 266         'ppc64le': 'powerpc64le',
 267         's390x': 's390x',
 268         'x64': 'x86_64',
 269         'x86': 'i686',
 270         'x86-64': 'x86_64',
 271         'x86_64': 'x86_64'
 272     }
 273
 274     # Consider the direct transformation first and then the special cases
 275     if cputype in cputype_mapper:
 276         cputype = cputype_mapper[cputype]
 277     elif cputype in {'xscale', 'arm'}:
 278         cputype = 'arm'
 279         if ostype == 'linux-android':
 280             ostype = 'linux-androideabi'
 281         elif ostype == 'unknown-freebsd':
 282             cputype = subprocess.check_output(
 283                 ['uname', '-p']).strip().decode(default_encoding)
 284             ostype = 'unknown-freebsd'
 285     elif cputype == 'armv6l':
 286         cputype = 'arm'
 287         if ostype == 'linux-android':
 288             ostype = 'linux-androideabi'
 289         else:
 290             ostype += 'eabihf'
 291     elif cputype in {'armv7l', 'armv8l'}:
 292         cputype = 'armv7'
 293         if ostype == 'linux-android':
 294             ostype = 'linux-androideabi'
 295         else:
 296             ostype += 'eabihf'
 297     elif cputype == 'mips':
 298         if sys.byteorder == 'big':
 299             cputype = 'mips'
 300         elif sys.byteorder == 'little':
 301             cputype = 'mipsel'
 302         else:
 303             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 304     elif cputype == 'mips64':
 305         if sys.byteorder == 'big':
 306             cputype = 'mips64'
 307         elif sys.byteorder == 'little':
 308             cputype = 'mips64el'
 309         else:
 310             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 311         # only the n64 ABI is supported, indicate it
 312         ostype += 'abi64'
 313     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 314         pass
 315     else:
 316         err = "unknown cpu type: {}".format(cputype)
 317         sys.exit(err)
 318
 319     return "{}-{}".format(cputype, ostype)
 320
 321
 322 @contextlib.contextmanager
 323 def output(filepath):
 324     tmp = filepath + '.tmp'
 325     with open(tmp, 'w') as f:
 326         yield f
 327     try:
 328         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 329         os.rename(tmp, filepath)
 330     except OSError:
 331         shutil.copy2(tmp, filepath)
 332         os.remove(tmp)
 333
 334
 335 class RustBuild(object):
 336     """Provide all the methods required to build Rust"""
 337     def __init__(self):
 338         self.cargo_channel = ''
 339         self.date = ''
 340         self._download_url = ''
 341         self.rustc_channel = ''
 342         self.rustfmt_channel = ''
 343         self.build = ''
 344         self.build_dir = ''
 345         self.clean = False
 346         self.config_toml = ''
 347         self.rust_root = ''
 348         self.use_locked_deps = ''
 349         self.use_vendored_sources = ''
 350         self.verbose = False
 351         self.git_version = None
 352         self.nix_deps_dir = None
 353
 354     def download_stage0(self):
 355         """Fetch the build system for Rust, written in Rust
 356
 357         This method will build a cache directory, then it will fetch the
 358         tarball which has the stage0 compiler used to then bootstrap the Rust
 359         compiler itself.
 360
 361         Each downloaded tarball is extracted, after that, the script
 362         will move all the content to the right place.
 363         """
 364         rustc_channel = self.rustc_channel
 365         cargo_channel = self.cargo_channel
 366         rustfmt_channel = self.rustfmt_channel
 367
 368         def support_xz():
 369             try:
 370                 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
 371                     temp_path = temp_file.name
 372                 with tarfile.open(temp_path, "w:xz"):
 373                     pass
 374                 return True
 375             except tarfile.CompressionError:
 376                 return False
 377
 378         if self.rustc().startswith(self.bin_root()) and \
 379                 (not os.path.exists(self.rustc()) or
 380                  self.program_out_of_date(self.rustc_stamp())):
 381             if os.path.exists(self.bin_root()):
 382                 shutil.rmtree(self.bin_root())
 383             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 384             filename = "rust-std-{}-{}{}".format(
 385                 rustc_channel, self.build, tarball_suffix)
 386             pattern = "rust-std-{}".format(self.build)
 387             self._download_stage0_helper(filename, pattern, tarball_suffix)
 388
 389             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 392             self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
 393             self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
 394             lib_dir = "{}/lib".format(self.bin_root())
 395             for lib in os.listdir(lib_dir):
 396                 if lib.endswith(".so"):
 397                     self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
 398             with output(self.rustc_stamp()) as rust_stamp:
 399                 rust_stamp.write(self.date)
 400
 401         if self.cargo().startswith(self.bin_root()) and \
 402                 (not os.path.exists(self.cargo()) or
 403                  self.program_out_of_date(self.cargo_stamp())):
 404             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 405             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 406                                               tarball_suffix)
 407             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 408             self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
 409             with output(self.cargo_stamp()) as cargo_stamp:
 410                 cargo_stamp.write(self.date)
 411
 412         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 413             not os.path.exists(self.rustfmt())
 414             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 415         ):
 416             if rustfmt_channel:
 417                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 418                 [channel, date] = rustfmt_channel.split('-', 1)
 419                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 420                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 421                 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
 422                 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
 423                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 424                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 425
 426     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 427         if date is None:
 428             date = self.date
 429         cache_dst = os.path.join(self.build_dir, "cache")
 430         rustc_cache = os.path.join(cache_dst, date)
 431         if not os.path.exists(rustc_cache):
 432             os.makedirs(rustc_cache)
 433
 434         url = "{}/dist/{}".format(self._download_url, date)
 435         tarball = os.path.join(rustc_cache, filename)
 436         if not os.path.exists(tarball):
 437             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 438         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 439
 440     def fix_bin_or_dylib(self, fname):
 441         """Modifies the interpreter section of 'fname' to fix the dynamic linker,
 442         or the RPATH section, to fix the dynamic library search path
 443
 444         This method is only required on NixOS and uses the PatchELF utility to
 445         change the interpreter/RPATH of ELF executables.
 446
 447         Please see https://nixos.org/patchelf.html for more information
 448         """
 449         default_encoding = sys.getdefaultencoding()
 450         try:
 451             ostype = subprocess.check_output(
 452                 ['uname', '-s']).strip().decode(default_encoding)
 453         except subprocess.CalledProcessError:
 454             return
 455         except OSError as reason:
 456             if getattr(reason, 'winerror', None) is not None:
 457                 return
 458             raise reason
 459
 460         if ostype != "Linux":
 461             return
 462
 463         if not os.path.exists("/etc/NIXOS"):
 464             return
 465         if os.path.exists("/lib"):
 466             return
 467
 468         # At this point we're pretty sure the user is running NixOS
 469         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 470         print(nix_os_msg, fname)
 471
 472         # Only build `stage0/.nix-deps` once.
 473         nix_deps_dir = self.nix_deps_dir
 474         if not nix_deps_dir:
 475             nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
 476             if not os.path.exists(nix_deps_dir):
 477                 os.makedirs(nix_deps_dir)
 478
 479             nix_deps = [
 480                 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
 481                 "stdenv.cc.bintools",
 482
 483                 # Needed as a system dependency of `libLLVM-*.so`.
 484                 "zlib",
 485
 486                 # Needed for patching ELF binaries (see doc comment above).
 487                 "patchelf",
 488             ]
 489
 490             # Run `nix-build` to "build" each dependency (which will likely reuse
 491             # the existing `/nix/store` copy, or at most download a pre-built copy).
 492             # Importantly, we don't rely on `nix-build` printing the `/nix/store`
 493             # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
 494             # ensuring garbage collection will never remove the `/nix/store` path
 495             # (which would break our patched binaries that hardcode those paths).
 496             for dep in nix_deps:
 497                 try:
 498                     subprocess.check_output([
 499                         "nix-build", "<nixpkgs>",
 500                         "-A", dep,
 501                         "-o", "{}/{}".format(nix_deps_dir, dep),
 502                     ])
 503                 except subprocess.CalledProcessError as reason:
 504                     print("warning: failed to call nix-build:", reason)
 505                     return
 506
 507             self.nix_deps_dir = nix_deps_dir
 508
 509         patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
 510
 511         if fname.endswith(".so"):
 512             # Dynamic library, patch RPATH to point to system dependencies.
 513             dylib_deps = ["zlib"]
 514             rpath_entries = [
 515                 # Relative default, all binary and dynamic libraries we ship
 516                 # appear to have this (even when `../lib` is redundant).
 517                 "$ORIGIN/../lib",
 518             ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
 519             patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
 520         else:
 521             bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
 522             with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
 523                 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
 524
 525         try:
 526             subprocess.check_output([patchelf] + patchelf_args + [fname])
 527         except subprocess.CalledProcessError as reason:
 528             print("warning: failed to call patchelf:", reason)
 529             return
 530
 531     def rustc_stamp(self):
 532         """Return the path for .rustc-stamp
 533
 534         >>> rb = RustBuild()
 535         >>> rb.build_dir = "build"
 536         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 537         True
 538         """
 539         return os.path.join(self.bin_root(), '.rustc-stamp')
 540
 541     def cargo_stamp(self):
 542         """Return the path for .cargo-stamp
 543
 544         >>> rb = RustBuild()
 545         >>> rb.build_dir = "build"
 546         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 547         True
 548         """
 549         return os.path.join(self.bin_root(), '.cargo-stamp')
 550
 551     def rustfmt_stamp(self):
 552         """Return the path for .rustfmt-stamp
 553
 554         >>> rb = RustBuild()
 555         >>> rb.build_dir = "build"
 556         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 557         True
 558         """
 559         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 560
 561     def program_out_of_date(self, stamp_path, extra=""):
 562         """Check if the given program stamp is out of date"""
 563         if not os.path.exists(stamp_path) or self.clean:
 564             return True
 565         with open(stamp_path, 'r') as stamp:
 566             return (self.date + extra) != stamp.read()
 567
 568     def bin_root(self):
 569         """Return the binary root directory
 570
 571         >>> rb = RustBuild()
 572         >>> rb.build_dir = "build"
 573         >>> rb.bin_root() == os.path.join("build", "stage0")
 574         True
 575
 576         When the 'build' property is given should be a nested directory:
 577
 578         >>> rb.build = "devel"
 579         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 580         True
 581         """
 582         return os.path.join(self.build_dir, self.build, "stage0")
 583
 584     def get_toml(self, key, section=None):
 585         """Returns the value of the given key in config.toml, otherwise returns None
 586
 587         >>> rb = RustBuild()
 588         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 589         >>> rb.get_toml("key2")
 590         'value2'
 591
 592         If the key does not exists, the result is None:
 593
 594         >>> rb.get_toml("key3") is None
 595         True
 596
 597         Optionally also matches the section the key appears in
 598
 599         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 600         >>> rb.get_toml('key', 'a')
 601         'value1'
 602         >>> rb.get_toml('key', 'b')
 603         'value2'
 604         >>> rb.get_toml('key', 'c') is None
 605         True
 606
 607         >>> rb.config_toml = 'key1 = true'
 608         >>> rb.get_toml("key1")
 609         'true'
 610         """
 611
 612         cur_section = None
 613         for line in self.config_toml.splitlines():
 614             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 615             if section_match is not None:
 616                 cur_section = section_match.group(1)
 617
 618             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 619             if match is not None:
 620                 value = match.group(1)
 621                 if section is None or section == cur_section:
 622                     return self.get_string(value) or value.strip()
 623         return None
 624
 625     def cargo(self):
 626         """Return config path for cargo"""
 627         return self.program_config('cargo')
 628
 629     def rustc(self):
 630         """Return config path for rustc"""
 631         return self.program_config('rustc')
 632
 633     def rustfmt(self):
 634         """Return config path for rustfmt"""
 635         if not self.rustfmt_channel:
 636             return None
 637         return self.program_config('rustfmt')
 638
 639     def program_config(self, program):
 640         """Return config path for the given program
 641
 642         >>> rb = RustBuild()
 643         >>> rb.config_toml = 'rustc = "rustc"\\n'
 644         >>> rb.program_config('rustc')
 645         'rustc'
 646         >>> rb.config_toml = ''
 647         >>> cargo_path = rb.program_config('cargo')
 648         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 649         ... "bin", "cargo")
 650         True
 651         """
 652         config = self.get_toml(program)
 653         if config:
 654             return os.path.expanduser(config)
 655         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 656             program, self.exe_suffix()))
 657
 658     @staticmethod
 659     def get_string(line):
 660         """Return the value between double quotes
 661
 662         >>> RustBuild.get_string('    "devel"   ')
 663         'devel'
 664         >>> RustBuild.get_string("    'devel'   ")
 665         'devel'
 666         >>> RustBuild.get_string('devel') is None
 667         True
 668         >>> RustBuild.get_string('    "devel   ')
 669         ''
 670         """
 671         start = line.find('"')
 672         if start != -1:
 673             end = start + 1 + line[start + 1:].find('"')
 674             return line[start + 1:end]
 675         start = line.find('\'')
 676         if start != -1:
 677             end = start + 1 + line[start + 1:].find('\'')
 678             return line[start + 1:end]
 679         return None
 680
 681     @staticmethod
 682     def exe_suffix():
 683         """Return a suffix for executables"""
 684         if sys.platform == 'win32':
 685             return '.exe'
 686         return ''
 687
 688     def bootstrap_binary(self):
 689         """Return the path of the bootstrap binary
 690
 691         >>> rb = RustBuild()
 692         >>> rb.build_dir = "build"
 693         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 694         ... "debug", "bootstrap")
 695         True
 696         """
 697         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 698
 699     def build_bootstrap(self):
 700         """Build bootstrap"""
 701         build_dir = os.path.join(self.build_dir, "bootstrap")
 702         if self.clean and os.path.exists(build_dir):
 703             shutil.rmtree(build_dir)
 704         env = os.environ.copy()
 705         # `CARGO_BUILD_TARGET` breaks bootstrap build.
 706         # See also: <https://github.com/rust-lang/rust/issues/70208>.
 707         if "CARGO_BUILD_TARGET" in env:
 708             del env["CARGO_BUILD_TARGET"]
 709         env["CARGO_TARGET_DIR"] = build_dir
 710         env["RUSTC"] = self.rustc()
 711         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 712             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 713             if "LD_LIBRARY_PATH" in env else ""
 714         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 715             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 716             if "DYLD_LIBRARY_PATH" in env else ""
 717         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 718             (os.pathsep + env["LIBRARY_PATH"]) \
 719             if "LIBRARY_PATH" in env else ""
 720         # preserve existing RUSTFLAGS
 721         env.setdefault("RUSTFLAGS", "")
 722         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 723
 724         build_section = "target.{}".format(self.build_triple())
 725         target_features = []
 726         if self.get_toml("crt-static", build_section) == "true":
 727             target_features += ["+crt-static"]
 728         elif self.get_toml("crt-static", build_section) == "false":
 729             target_features += ["-crt-static"]
 730         if target_features:
 731             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 732         target_linker = self.get_toml("linker", build_section)
 733         if target_linker is not None:
 734             env["RUSTFLAGS"] += " -C linker=" + target_linker
 735         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 736         if self.get_toml("deny-warnings", "rust") != "false":
 737             env["RUSTFLAGS"] += " -Dwarnings"
 738
 739         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 740             os.pathsep + env["PATH"]
 741         if not os.path.isfile(self.cargo()):
 742             raise Exception("no cargo executable found at `{}`".format(
 743                 self.cargo()))
 744         args = [self.cargo(), "build", "--manifest-path",
 745                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 746         for _ in range(1, self.verbose):
 747             args.append("--verbose")
 748         if self.use_locked_deps:
 749             args.append("--locked")
 750         if self.use_vendored_sources:
 751             args.append("--frozen")
 752         run(args, env=env, verbose=self.verbose)
 753
 754     def build_triple(self):
 755         """Build triple as in LLVM"""
 756         config = self.get_toml('build')
 757         if config:
 758             return config
 759         return default_build_triple()
 760
 761     def check_submodule(self, module, slow_submodules):
 762         if not slow_submodules:
 763             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 764                                            cwd=os.path.join(self.rust_root, module),
 765                                            stdout=subprocess.PIPE)
 766             return checked_out
 767         else:
 768             return None
 769
 770     def update_submodule(self, module, checked_out, recorded_submodules):
 771         module_path = os.path.join(self.rust_root, module)
 772
 773         if checked_out is not None:
 774             default_encoding = sys.getdefaultencoding()
 775             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 776             if recorded_submodules[module] == checked_out:
 777                 return
 778
 779         print("Updating submodule", module)
 780
 781         run(["git", "submodule", "-q", "sync", module],
 782             cwd=self.rust_root, verbose=self.verbose)
 783
 784         update_args = ["git", "submodule", "update", "--init", "--recursive"]
 785         if self.git_version >= distutils.version.LooseVersion("2.11.0"):
 786             update_args.append("--progress")
 787         update_args.append(module)
 788         run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
 789
 790         run(["git", "reset", "-q", "--hard"],
 791             cwd=module_path, verbose=self.verbose)
 792         run(["git", "clean", "-qdfx"],
 793             cwd=module_path, verbose=self.verbose)
 794
 795     def update_submodules(self):
 796         """Update submodules"""
 797         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 798                 self.get_toml('submodules') == "false":
 799             return
 800
 801         default_encoding = sys.getdefaultencoding()
 802
 803         # check the existence and version of 'git' command
 804         git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
 805         self.git_version = distutils.version.LooseVersion(git_version_str)
 806
 807         slow_submodules = self.get_toml('fast-submodules') == "false"
 808         start_time = time()
 809         if slow_submodules:
 810             print('Unconditionally updating all submodules')
 811         else:
 812             print('Updating only changed submodules')
 813         default_encoding = sys.getdefaultencoding()
 814         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 815             ["git", "config", "--file",
 816              os.path.join(self.rust_root, ".gitmodules"),
 817              "--get-regexp", "path"]
 818         ).decode(default_encoding).splitlines()]
 819         filtered_submodules = []
 820         submodules_names = []
 821         for module in submodules:
 822             if module.endswith("llvm-project"):
 823                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 824                     continue
 825             check = self.check_submodule(module, slow_submodules)
 826             filtered_submodules.append((module, check))
 827             submodules_names.append(module)
 828         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 829                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 830         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 831         recorded_submodules = {}
 832         for data in recorded:
 833             data = data.split()
 834             recorded_submodules[data[3]] = data[2]
 835         for module in filtered_submodules:
 836             self.update_submodule(module[0], module[1], recorded_submodules)
 837         print("Submodules updated in %.2f seconds" % (time() - start_time))
 838
 839     def set_normal_environment(self):
 840         """Set download URL for normal environment"""
 841         if 'RUSTUP_DIST_SERVER' in os.environ:
 842             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 843         else:
 844             self._download_url = 'https://static.rust-lang.org'
 845
 846     def set_dev_environment(self):
 847         """Set download URL for development environment"""
 848         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 849             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 850         else:
 851             self._download_url = 'https://dev-static.rust-lang.org'
 852
 853     def check_vendored_status(self):
 854         """Check that vendoring is configured properly"""
 855         vendor_dir = os.path.join(self.rust_root, 'vendor')
 856         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 857             if os.environ.get('USER') != os.environ['SUDO_USER']:
 858                 self.use_vendored_sources = True
 859                 print('info: looks like you are running this command under `sudo`')
 860                 print('      and so in order to preserve your $HOME this will now')
 861                 print('      use vendored sources by default.')
 862                 if not os.path.exists(vendor_dir):
 863                     print('error: vendoring required, but vendor directory does not exist.')
 864                     print('       Run `cargo vendor` without sudo to initialize the '
 865                           'vendor directory.')
 866                     raise Exception("{} not found".format(vendor_dir))
 867
 868         if self.use_vendored_sources:
 869             if not os.path.exists('.cargo'):
 870                 os.makedirs('.cargo')
 871             with output('.cargo/config') as cargo_config:
 872                 cargo_config.write(
 873                     "[source.crates-io]\n"
 874                     "replace-with = 'vendored-sources'\n"
 875                     "registry = 'https://example.com'\n"
 876                     "\n"
 877                     "[source.vendored-sources]\n"
 878                     "directory = '{}/vendor'\n"
 879                     .format(self.rust_root))
 880         else:
 881             if os.path.exists('.cargo'):
 882                 shutil.rmtree('.cargo')
 883
 884     def ensure_vendored(self):
 885         """Ensure that the vendored sources are available if needed"""
 886         vendor_dir = os.path.join(self.rust_root, 'vendor')
 887         # Note that this does not handle updating the vendored dependencies if
 888         # the rust git repository is updated. Normal development usually does
 889         # not use vendoring, so hopefully this isn't too much of a problem.
 890         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 891             run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
 892                 verbose=self.verbose, cwd=self.rust_root)
 893
 894
 895 def bootstrap(help_triggered):
 896     """Configure, fetch, build and run the initial bootstrap"""
 897
 898     # If the user is asking for help, let them know that the whole download-and-build
 899     # process has to happen before anything is printed out.
 900     if help_triggered:
 901         print("info: Downloading and building bootstrap before processing --help")
 902         print("      command. See src/bootstrap/README.md for help with common")
 903         print("      commands.")
 904
 905     parser = argparse.ArgumentParser(description='Build rust')
 906     parser.add_argument('--config')
 907     parser.add_argument('--build')
 908     parser.add_argument('--src')
 909     parser.add_argument('--clean', action='store_true')
 910     parser.add_argument('-v', '--verbose', action='count', default=0)
 911
 912     args = [a for a in sys.argv if a != '-h' and a != '--help']
 913     args, _ = parser.parse_known_args(args)
 914
 915     # Configure initial bootstrap
 916     build = RustBuild()
 917     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 918     build.verbose = args.verbose
 919     build.clean = args.clean
 920
 921     # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
 922     # exists).
 923     toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
 924     if not toml_path and os.path.exists('config.toml'):
 925         toml_path = 'config.toml'
 926
 927     if toml_path:
 928         if not os.path.exists(toml_path):
 929             toml_path = os.path.join(build.rust_root, toml_path)
 930
 931         with open(toml_path) as config:
 932             build.config_toml = config.read()
 933
 934     config_verbose = build.get_toml('verbose', 'build')
 935     if config_verbose is not None:
 936         build.verbose = max(build.verbose, int(config_verbose))
 937
 938     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 939
 940     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 941
 942     build.check_vendored_status()
 943
 944     build_dir = build.get_toml('build-dir', 'build') or 'build'
 945     build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
 946
 947     data = stage0_data(build.rust_root)
 948     build.date = data['date']
 949     build.rustc_channel = data['rustc']
 950     build.cargo_channel = data['cargo']
 951
 952     if "rustfmt" in data:
 953         build.rustfmt_channel = data['rustfmt']
 954
 955     if 'dev' in data:
 956         build.set_dev_environment()
 957     else:
 958         build.set_normal_environment()
 959
 960     build.update_submodules()
 961
 962     # Fetch/build the bootstrap
 963     build.build = args.build or build.build_triple()
 964     build.download_stage0()
 965     sys.stdout.flush()
 966     build.ensure_vendored()
 967     build.build_bootstrap()
 968     sys.stdout.flush()
 969
 970     # Run the bootstrap
 971     args = [build.bootstrap_binary()]
 972     args.extend(sys.argv[1:])
 973     env = os.environ.copy()
 974     env["BUILD"] = build.build
 975     env["SRC"] = build.rust_root
 976     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 977     env["BOOTSTRAP_PYTHON"] = sys.executable
 978     env["BUILD_DIR"] = build.build_dir
 979     env["RUSTC_BOOTSTRAP"] = '1'
 980     env["CARGO"] = build.cargo()
 981     env["RUSTC"] = build.rustc()
 982     if toml_path:
 983         env["BOOTSTRAP_CONFIG"] = toml_path
 984     if build.rustfmt():
 985         env["RUSTFMT"] = build.rustfmt()
 986     run(args, env=env, verbose=build.verbose)
 987
 988
 989 def main():
 990     """Entry point for the bootstrap process"""
 991     start_time = time()
 992
 993     # x.py help <cmd> ...
 994     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 995         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 996
 997     help_triggered = (
 998         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 999     try:
1000         bootstrap(help_triggered)
1001         if not help_triggered:
1002             print("Build completed successfully in {}".format(
1003                 format_build_time(time() - start_time)))
1004     except (SystemExit, KeyboardInterrupt) as error:
1005         if hasattr(error, 'code') and isinstance(error.code, int):
1006             exit_code = error.code
1007         else:
1008             exit_code = 1
1009             print(error)
1010         if not help_triggered:
1011             print("Build completed unsuccessfully in {}".format(
1012                 format_build_time(time() - start_time)))
1013         sys.exit(exit_code)
1014
1015
1016 if __name__ == '__main__':
1017     main()