src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(".tar.gz", "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = ''
 324         self.rustc_channel = ''
 325         self.build = ''
 326         self.build_dir = os.path.join(os.getcwd(), "build")
 327         self.clean = False
 328         self.config_toml = ''
 329         self.rust_root = ''
 330         self.use_locked_deps = ''
 331         self.use_vendored_sources = ''
 332         self.verbose = False
 333
 334     def download_stage0(self):
 335         """Fetch the build system for Rust, written in Rust
 336
 337         This method will build a cache directory, then it will fetch the
 338         tarball which has the stage0 compiler used to then bootstrap the Rust
 339         compiler itself.
 340
 341         Each downloaded tarball is extracted, after that, the script
 342         will move all the content to the right place.
 343         """
 344         rustc_channel = self.rustc_channel
 345         cargo_channel = self.cargo_channel
 346
 347         if self.rustc().startswith(self.bin_root()) and \
 348                 (not os.path.exists(self.rustc()) or
 349                  self.program_out_of_date(self.rustc_stamp())):
 350             if os.path.exists(self.bin_root()):
 351                 shutil.rmtree(self.bin_root())
 352             filename = "rust-std-{}-{}.tar.gz".format(
 353                 rustc_channel, self.build)
 354             pattern = "rust-std-{}".format(self.build)
 355             self._download_stage0_helper(filename, pattern)
 356
 357             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
 358             self._download_stage0_helper(filename, "rustc")
 359             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 360             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 361             with output(self.rustc_stamp()) as rust_stamp:
 362                 rust_stamp.write(self.date)
 363
 364             # This is required so that we don't mix incompatible MinGW
 365             # libraries/binaries that are included in rust-std with
 366             # the system MinGW ones.
 367             if "pc-windows-gnu" in self.build:
 368                 filename = "rust-mingw-{}-{}.tar.gz".format(
 369                     rustc_channel, self.build)
 370                 self._download_stage0_helper(filename, "rust-mingw")
 371
 372         if self.cargo().startswith(self.bin_root()) and \
 373                 (not os.path.exists(self.cargo()) or
 374                  self.program_out_of_date(self.cargo_stamp())):
 375             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
 376             self._download_stage0_helper(filename, "cargo")
 377             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 378             with output(self.cargo_stamp()) as cargo_stamp:
 379                 cargo_stamp.write(self.date)
 380
 381     def _download_stage0_helper(self, filename, pattern):
 382         cache_dst = os.path.join(self.build_dir, "cache")
 383         rustc_cache = os.path.join(cache_dst, self.date)
 384         if not os.path.exists(rustc_cache):
 385             os.makedirs(rustc_cache)
 386
 387         url = "{}/dist/{}".format(self._download_url, self.date)
 388         tarball = os.path.join(rustc_cache, filename)
 389         if not os.path.exists(tarball):
 390             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 391         unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
 392
 393     @staticmethod
 394     def fix_executable(fname):
 395         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 396
 397         This method is only required on NixOS and uses the PatchELF utility to
 398         change the dynamic linker of ELF executables.
 399
 400         Please see https://nixos.org/patchelf.html for more information
 401         """
 402         default_encoding = sys.getdefaultencoding()
 403         try:
 404             ostype = subprocess.check_output(
 405                 ['uname', '-s']).strip().decode(default_encoding)
 406         except subprocess.CalledProcessError:
 407             return
 408         except OSError as reason:
 409             if getattr(reason, 'winerror', None) is not None:
 410                 return
 411             raise reason
 412
 413         if ostype != "Linux":
 414             return
 415
 416         if not os.path.exists("/etc/NIXOS"):
 417             return
 418         if os.path.exists("/lib"):
 419             return
 420
 421         # At this point we're pretty sure the user is running NixOS
 422         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 423         print(nix_os_msg, fname)
 424
 425         try:
 426             interpreter = subprocess.check_output(
 427                 ["patchelf", "--print-interpreter", fname])
 428             interpreter = interpreter.strip().decode(default_encoding)
 429         except subprocess.CalledProcessError as reason:
 430             print("warning: failed to call patchelf:", reason)
 431             return
 432
 433         loader = interpreter.split("/")[-1]
 434
 435         try:
 436             ldd_output = subprocess.check_output(
 437                 ['ldd', '/run/current-system/sw/bin/sh'])
 438             ldd_output = ldd_output.strip().decode(default_encoding)
 439         except subprocess.CalledProcessError as reason:
 440             print("warning: unable to call ldd:", reason)
 441             return
 442
 443         for line in ldd_output.splitlines():
 444             libname = line.split()[0]
 445             if libname.endswith(loader):
 446                 loader_path = libname[:len(libname) - len(loader)]
 447                 break
 448         else:
 449             print("warning: unable to find the path to the dynamic linker")
 450             return
 451
 452         correct_interpreter = loader_path + loader
 453
 454         try:
 455             subprocess.check_output(
 456                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 457         except subprocess.CalledProcessError as reason:
 458             print("warning: failed to call patchelf:", reason)
 459             return
 460
 461     def rustc_stamp(self):
 462         """Return the path for .rustc-stamp
 463
 464         >>> rb = RustBuild()
 465         >>> rb.build_dir = "build"
 466         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 467         True
 468         """
 469         return os.path.join(self.bin_root(), '.rustc-stamp')
 470
 471     def cargo_stamp(self):
 472         """Return the path for .cargo-stamp
 473
 474         >>> rb = RustBuild()
 475         >>> rb.build_dir = "build"
 476         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 477         True
 478         """
 479         return os.path.join(self.bin_root(), '.cargo-stamp')
 480
 481     def program_out_of_date(self, stamp_path):
 482         """Check if the given program stamp is out of date"""
 483         if not os.path.exists(stamp_path) or self.clean:
 484             return True
 485         with open(stamp_path, 'r') as stamp:
 486             return self.date != stamp.read()
 487
 488     def bin_root(self):
 489         """Return the binary root directory
 490
 491         >>> rb = RustBuild()
 492         >>> rb.build_dir = "build"
 493         >>> rb.bin_root() == os.path.join("build", "stage0")
 494         True
 495
 496         When the 'build' property is given should be a nested directory:
 497
 498         >>> rb.build = "devel"
 499         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 500         True
 501         """
 502         return os.path.join(self.build_dir, self.build, "stage0")
 503
 504     def get_toml(self, key, section=None):
 505         """Returns the value of the given key in config.toml, otherwise returns None
 506
 507         >>> rb = RustBuild()
 508         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 509         >>> rb.get_toml("key2")
 510         'value2'
 511
 512         If the key does not exists, the result is None:
 513
 514         >>> rb.get_toml("key3") is None
 515         True
 516
 517         Optionally also matches the section the key appears in
 518
 519         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 520         >>> rb.get_toml('key', 'a')
 521         'value1'
 522         >>> rb.get_toml('key', 'b')
 523         'value2'
 524         >>> rb.get_toml('key', 'c') is None
 525         True
 526
 527         >>> rb.config_toml = 'key1 = true'
 528         >>> rb.get_toml("key1")
 529         'true'
 530         """
 531
 532         cur_section = None
 533         for line in self.config_toml.splitlines():
 534             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 535             if section_match is not None:
 536                 cur_section = section_match.group(1)
 537
 538             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 539             if match is not None:
 540                 value = match.group(1)
 541                 if section is None or section == cur_section:
 542                     return self.get_string(value) or value.strip()
 543         return None
 544
 545     def cargo(self):
 546         """Return config path for cargo"""
 547         return self.program_config('cargo')
 548
 549     def rustc(self):
 550         """Return config path for rustc"""
 551         return self.program_config('rustc')
 552
 553     def program_config(self, program):
 554         """Return config path for the given program
 555
 556         >>> rb = RustBuild()
 557         >>> rb.config_toml = 'rustc = "rustc"\\n'
 558         >>> rb.program_config('rustc')
 559         'rustc'
 560         >>> rb.config_toml = ''
 561         >>> cargo_path = rb.program_config('cargo')
 562         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 563         ... "bin", "cargo")
 564         True
 565         """
 566         config = self.get_toml(program)
 567         if config:
 568             return os.path.expanduser(config)
 569         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 570             program, self.exe_suffix()))
 571
 572     @staticmethod
 573     def get_string(line):
 574         """Return the value between double quotes
 575
 576         >>> RustBuild.get_string('    "devel"   ')
 577         'devel'
 578         >>> RustBuild.get_string("    'devel'   ")
 579         'devel'
 580         >>> RustBuild.get_string('devel') is None
 581         True
 582         >>> RustBuild.get_string('    "devel   ')
 583         ''
 584         """
 585         start = line.find('"')
 586         if start != -1:
 587             end = start + 1 + line[start + 1:].find('"')
 588             return line[start + 1:end]
 589         start = line.find('\'')
 590         if start != -1:
 591             end = start + 1 + line[start + 1:].find('\'')
 592             return line[start + 1:end]
 593         return None
 594
 595     @staticmethod
 596     def exe_suffix():
 597         """Return a suffix for executables"""
 598         if sys.platform == 'win32':
 599             return '.exe'
 600         return ''
 601
 602     def bootstrap_binary(self):
 603         """Return the path of the bootstrap binary
 604
 605         >>> rb = RustBuild()
 606         >>> rb.build_dir = "build"
 607         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 608         ... "debug", "bootstrap")
 609         True
 610         """
 611         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 612
 613     def build_bootstrap(self):
 614         """Build bootstrap"""
 615         build_dir = os.path.join(self.build_dir, "bootstrap")
 616         if self.clean and os.path.exists(build_dir):
 617             shutil.rmtree(build_dir)
 618         env = os.environ.copy()
 619         env["RUSTC_BOOTSTRAP"] = '1'
 620         env["CARGO_TARGET_DIR"] = build_dir
 621         env["RUSTC"] = self.rustc()
 622         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 623             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 624             if "LD_LIBRARY_PATH" in env else ""
 625         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 626             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 627             if "DYLD_LIBRARY_PATH" in env else ""
 628         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 629             (os.pathsep + env["LIBRARY_PATH"]) \
 630             if "LIBRARY_PATH" in env else ""
 631         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 632
 633         build_section = "target.{}".format(self.build_triple())
 634         target_features = []
 635         if self.get_toml("crt-static", build_section) == "true":
 636             target_features += ["+crt-static"]
 637         elif self.get_toml("crt-static", build_section) == "false":
 638             target_features += ["-crt-static"]
 639         if target_features:
 640             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 641         target_linker = self.get_toml("linker", build_section)
 642         if target_linker is not None:
 643             env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
 644         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes "
 645         if self.get_toml("deny-warnings", "rust") != "false":
 646             env["RUSTFLAGS"] += "-Dwarnings "
 647
 648         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 649             os.pathsep + env["PATH"]
 650         if not os.path.isfile(self.cargo()):
 651             raise Exception("no cargo executable found at `{}`".format(
 652                 self.cargo()))
 653         args = [self.cargo(), "build", "--manifest-path",
 654                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 655         for _ in range(1, self.verbose):
 656             args.append("--verbose")
 657         if self.use_locked_deps:
 658             args.append("--locked")
 659         if self.use_vendored_sources:
 660             args.append("--frozen")
 661         run(args, env=env, verbose=self.verbose)
 662
 663     def build_triple(self):
 664         """Build triple as in LLVM"""
 665         config = self.get_toml('build')
 666         if config:
 667             return config
 668         return default_build_triple()
 669
 670     def check_submodule(self, module, slow_submodules):
 671         if not slow_submodules:
 672             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 673                                            cwd=os.path.join(self.rust_root, module),
 674                                            stdout=subprocess.PIPE)
 675             return checked_out
 676         else:
 677             return None
 678
 679     def update_submodule(self, module, checked_out, recorded_submodules):
 680         module_path = os.path.join(self.rust_root, module)
 681
 682         if checked_out is not None:
 683             default_encoding = sys.getdefaultencoding()
 684             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 685             if recorded_submodules[module] == checked_out:
 686                 return
 687
 688         print("Updating submodule", module)
 689
 690         run(["git", "submodule", "-q", "sync", module],
 691             cwd=self.rust_root, verbose=self.verbose)
 692         try:
 693             run(["git", "submodule", "update",
 694                  "--init", "--recursive", "--progress", module],
 695                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 696         except RuntimeError:
 697             # Some versions of git don't support --progress.
 698             run(["git", "submodule", "update",
 699                  "--init", "--recursive", module],
 700                 cwd=self.rust_root, verbose=self.verbose)
 701         run(["git", "reset", "-q", "--hard"],
 702             cwd=module_path, verbose=self.verbose)
 703         run(["git", "clean", "-qdfx"],
 704             cwd=module_path, verbose=self.verbose)
 705
 706     def update_submodules(self):
 707         """Update submodules"""
 708         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 709                 self.get_toml('submodules') == "false":
 710             return
 711
 712         # check the existence of 'git' command
 713         try:
 714             subprocess.check_output(['git', '--version'])
 715         except (subprocess.CalledProcessError, OSError):
 716             print("error: `git` is not found, please make sure it's installed and in the path.")
 717             sys.exit(1)
 718
 719         slow_submodules = self.get_toml('fast-submodules') == "false"
 720         start_time = time()
 721         if slow_submodules:
 722             print('Unconditionally updating all submodules')
 723         else:
 724             print('Updating only changed submodules')
 725         default_encoding = sys.getdefaultencoding()
 726         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 727             ["git", "config", "--file",
 728              os.path.join(self.rust_root, ".gitmodules"),
 729              "--get-regexp", "path"]
 730         ).decode(default_encoding).splitlines()]
 731         filtered_submodules = []
 732         submodules_names = []
 733         for module in submodules:
 734             if module.endswith("llvm-project"):
 735                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 736                     continue
 737             if module.endswith("llvm-emscripten"):
 738                 backends = self.get_toml('codegen-backends')
 739                 if backends is None or not 'emscripten' in backends:
 740                     continue
 741             check = self.check_submodule(module, slow_submodules)
 742             filtered_submodules.append((module, check))
 743             submodules_names.append(module)
 744         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 745                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 746         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 747         recorded_submodules = {}
 748         for data in recorded:
 749             data = data.split()
 750             recorded_submodules[data[3]] = data[2]
 751         for module in filtered_submodules:
 752             self.update_submodule(module[0], module[1], recorded_submodules)
 753         print("Submodules updated in %.2f seconds" % (time() - start_time))
 754
 755     def set_normal_environment(self):
 756         """Set download URL for normal environment"""
 757         if 'RUSTUP_DIST_SERVER' in os.environ:
 758             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 759         else:
 760             self._download_url = 'https://static.rust-lang.org'
 761
 762     def set_dev_environment(self):
 763         """Set download URL for development environment"""
 764         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 765             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 766         else:
 767             self._download_url = 'https://dev-static.rust-lang.org'
 768
 769     def check_vendored_status(self):
 770         """Check that vendoring is configured properly"""
 771         vendor_dir = os.path.join(self.rust_root, 'vendor')
 772         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 773             if os.environ.get('USER') != os.environ['SUDO_USER']:
 774                 self.use_vendored_sources = True
 775                 print('info: looks like you are running this command under `sudo`')
 776                 print('      and so in order to preserve your $HOME this will now')
 777                 print('      use vendored sources by default.')
 778                 if not os.path.exists(vendor_dir):
 779                     print('error: vendoring required, but vendor directory does not exist.')
 780                     print('       Run `cargo vendor` without sudo to initialize the '
 781                         'vendor directory.')
 782                     raise Exception("{} not found".format(vendor_dir))
 783
 784         if self.use_vendored_sources:
 785             if not os.path.exists('.cargo'):
 786                 os.makedirs('.cargo')
 787             with output('.cargo/config') as cargo_config:
 788                 cargo_config.write(
 789                     "[source.crates-io]\n"
 790                     "replace-with = 'vendored-sources'\n"
 791                     "registry = 'https://example.com'\n"
 792                     "\n"
 793                     "[source.vendored-sources]\n"
 794                     "directory = '{}/vendor'\n"
 795                 .format(self.rust_root))
 796         else:
 797             if os.path.exists('.cargo'):
 798                 shutil.rmtree('.cargo')
 799
 800     def ensure_vendored(self):
 801         """Ensure that the vendored sources are available if needed"""
 802         vendor_dir = os.path.join(self.rust_root, 'vendor')
 803         # Note that this does not handle updating the vendored dependencies if
 804         # the rust git repository is updated. Normal development usually does
 805         # not use vendoring, so hopefully this isn't too much of a problem.
 806         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 807             run([self.cargo(), "vendor"],
 808                 verbose=self.verbose, cwd=self.rust_root)
 809
 810
 811 def bootstrap(help_triggered):
 812     """Configure, fetch, build and run the initial bootstrap"""
 813
 814     # If the user is asking for help, let them know that the whole download-and-build
 815     # process has to happen before anything is printed out.
 816     if help_triggered:
 817         print("info: Downloading and building bootstrap before processing --help")
 818         print("      command. See src/bootstrap/README.md for help with common")
 819         print("      commands.")
 820
 821     parser = argparse.ArgumentParser(description='Build rust')
 822     parser.add_argument('--config')
 823     parser.add_argument('--build')
 824     parser.add_argument('--src')
 825     parser.add_argument('--clean', action='store_true')
 826     parser.add_argument('-v', '--verbose', action='count', default=0)
 827
 828     args = [a for a in sys.argv if a != '-h' and a != '--help']
 829     args, _ = parser.parse_known_args(args)
 830
 831     # Configure initial bootstrap
 832     build = RustBuild()
 833     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 834     build.verbose = args.verbose
 835     build.clean = args.clean
 836
 837     try:
 838         with open(args.config or 'config.toml') as config:
 839             build.config_toml = config.read()
 840     except (OSError, IOError):
 841         pass
 842
 843     config_verbose = build.get_toml('verbose', 'build')
 844     if config_verbose is not None:
 845         build.verbose = max(build.verbose, int(config_verbose))
 846
 847     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 848
 849     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 850
 851     build.check_vendored_status()
 852
 853     data = stage0_data(build.rust_root)
 854     build.date = data['date']
 855     build.rustc_channel = data['rustc']
 856     build.cargo_channel = data['cargo']
 857
 858     if 'dev' in data:
 859         build.set_dev_environment()
 860     else:
 861         build.set_normal_environment()
 862
 863     build.update_submodules()
 864
 865     # Fetch/build the bootstrap
 866     build.build = args.build or build.build_triple()
 867     build.download_stage0()
 868     sys.stdout.flush()
 869     build.ensure_vendored()
 870     build.build_bootstrap()
 871     sys.stdout.flush()
 872
 873     # Run the bootstrap
 874     args = [build.bootstrap_binary()]
 875     args.extend(sys.argv[1:])
 876     env = os.environ.copy()
 877     env["BUILD"] = build.build
 878     env["SRC"] = build.rust_root
 879     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 880     env["BOOTSTRAP_PYTHON"] = sys.executable
 881     env["BUILD_DIR"] = build.build_dir
 882     env["RUSTC_BOOTSTRAP"] = '1'
 883     env["CARGO"] = build.cargo()
 884     env["RUSTC"] = build.rustc()
 885     run(args, env=env, verbose=build.verbose)
 886
 887
 888 def main():
 889     """Entry point for the bootstrap process"""
 890     start_time = time()
 891
 892     # x.py help <cmd> ...
 893     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 894         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 895
 896     help_triggered = (
 897         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 898     try:
 899         bootstrap(help_triggered)
 900         if not help_triggered:
 901             print("Build completed successfully in {}".format(
 902                 format_build_time(time() - start_time)))
 903     except (SystemExit, KeyboardInterrupt) as error:
 904         if hasattr(error, 'code') and isinstance(error.code, int):
 905             exit_code = error.code
 906         else:
 907             exit_code = 1
 908             print(error)
 909         if not help_triggered:
 910             print("Build completed unsuccessfully in {}".format(
 911                 format_build_time(time() - start_time)))
 912         sys.exit(exit_code)
 913
 914
 915 if __name__ == '__main__':
 916     main()