src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(".tar.gz", "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = ''
 324         self.rustc_channel = ''
 325         self.build = ''
 326         self.build_dir = os.path.join(os.getcwd(), "build")
 327         self.clean = False
 328         self.config_toml = ''
 329         self.rust_root = ''
 330         self.use_locked_deps = ''
 331         self.use_vendored_sources = ''
 332         self.verbose = False
 333
 334     def download_stage0(self):
 335         """Fetch the build system for Rust, written in Rust
 336
 337         This method will build a cache directory, then it will fetch the
 338         tarball which has the stage0 compiler used to then bootstrap the Rust
 339         compiler itself.
 340
 341         Each downloaded tarball is extracted, after that, the script
 342         will move all the content to the right place.
 343         """
 344         rustc_channel = self.rustc_channel
 345         cargo_channel = self.cargo_channel
 346
 347         if self.rustc().startswith(self.bin_root()) and \
 348                 (not os.path.exists(self.rustc()) or
 349                  self.program_out_of_date(self.rustc_stamp())):
 350             if os.path.exists(self.bin_root()):
 351                 shutil.rmtree(self.bin_root())
 352             filename = "rust-std-{}-{}.tar.gz".format(
 353                 rustc_channel, self.build)
 354             pattern = "rust-std-{}".format(self.build)
 355             self._download_stage0_helper(filename, pattern)
 356
 357             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
 358             self._download_stage0_helper(filename, "rustc")
 359             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 360             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 361             with output(self.rustc_stamp()) as rust_stamp:
 362                 rust_stamp.write(self.date)
 363
 364             # This is required so that we don't mix incompatible MinGW
 365             # libraries/binaries that are included in rust-std with
 366             # the system MinGW ones.
 367             if "pc-windows-gnu" in self.build:
 368                 filename = "rust-mingw-{}-{}.tar.gz".format(
 369                     rustc_channel, self.build)
 370                 self._download_stage0_helper(filename, "rust-mingw")
 371
 372         if self.cargo().startswith(self.bin_root()) and \
 373                 (not os.path.exists(self.cargo()) or
 374                  self.program_out_of_date(self.cargo_stamp())):
 375             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
 376             self._download_stage0_helper(filename, "cargo")
 377             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 378             with output(self.cargo_stamp()) as cargo_stamp:
 379                 cargo_stamp.write(self.date)
 380
 381     def _download_stage0_helper(self, filename, pattern):
 382         cache_dst = os.path.join(self.build_dir, "cache")
 383         rustc_cache = os.path.join(cache_dst, self.date)
 384         if not os.path.exists(rustc_cache):
 385             os.makedirs(rustc_cache)
 386
 387         url = "{}/dist/{}".format(self._download_url, self.date)
 388         tarball = os.path.join(rustc_cache, filename)
 389         if not os.path.exists(tarball):
 390             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 391         unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
 392
 393     @staticmethod
 394     def fix_executable(fname):
 395         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 396
 397         This method is only required on NixOS and uses the PatchELF utility to
 398         change the dynamic linker of ELF executables.
 399
 400         Please see https://nixos.org/patchelf.html for more information
 401         """
 402         default_encoding = sys.getdefaultencoding()
 403         try:
 404             ostype = subprocess.check_output(
 405                 ['uname', '-s']).strip().decode(default_encoding)
 406         except subprocess.CalledProcessError:
 407             return
 408         except OSError as reason:
 409             if getattr(reason, 'winerror', None) is not None:
 410                 return
 411             raise reason
 412
 413         if ostype != "Linux":
 414             return
 415
 416         if not os.path.exists("/etc/NIXOS"):
 417             return
 418         if os.path.exists("/lib"):
 419             return
 420
 421         # At this point we're pretty sure the user is running NixOS
 422         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 423         print(nix_os_msg, fname)
 424
 425         try:
 426             interpreter = subprocess.check_output(
 427                 ["patchelf", "--print-interpreter", fname])
 428             interpreter = interpreter.strip().decode(default_encoding)
 429         except subprocess.CalledProcessError as reason:
 430             print("warning: failed to call patchelf:", reason)
 431             return
 432
 433         loader = interpreter.split("/")[-1]
 434
 435         try:
 436             ldd_output = subprocess.check_output(
 437                 ['ldd', '/run/current-system/sw/bin/sh'])
 438             ldd_output = ldd_output.strip().decode(default_encoding)
 439         except subprocess.CalledProcessError as reason:
 440             print("warning: unable to call ldd:", reason)
 441             return
 442
 443         for line in ldd_output.splitlines():
 444             libname = line.split()[0]
 445             if libname.endswith(loader):
 446                 loader_path = libname[:len(libname) - len(loader)]
 447                 break
 448         else:
 449             print("warning: unable to find the path to the dynamic linker")
 450             return
 451
 452         correct_interpreter = loader_path + loader
 453
 454         try:
 455             subprocess.check_output(
 456                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 457         except subprocess.CalledProcessError as reason:
 458             print("warning: failed to call patchelf:", reason)
 459             return
 460
 461     def rustc_stamp(self):
 462         """Return the path for .rustc-stamp
 463
 464         >>> rb = RustBuild()
 465         >>> rb.build_dir = "build"
 466         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 467         True
 468         """
 469         return os.path.join(self.bin_root(), '.rustc-stamp')
 470
 471     def cargo_stamp(self):
 472         """Return the path for .cargo-stamp
 473
 474         >>> rb = RustBuild()
 475         >>> rb.build_dir = "build"
 476         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 477         True
 478         """
 479         return os.path.join(self.bin_root(), '.cargo-stamp')
 480
 481     def program_out_of_date(self, stamp_path):
 482         """Check if the given program stamp is out of date"""
 483         if not os.path.exists(stamp_path) or self.clean:
 484             return True
 485         with open(stamp_path, 'r') as stamp:
 486             return self.date != stamp.read()
 487
 488     def bin_root(self):
 489         """Return the binary root directory
 490
 491         >>> rb = RustBuild()
 492         >>> rb.build_dir = "build"
 493         >>> rb.bin_root() == os.path.join("build", "stage0")
 494         True
 495
 496         When the 'build' property is given should be a nested directory:
 497
 498         >>> rb.build = "devel"
 499         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 500         True
 501         """
 502         return os.path.join(self.build_dir, self.build, "stage0")
 503
 504     def get_toml(self, key, section=None):
 505         """Returns the value of the given key in config.toml, otherwise returns None
 506
 507         >>> rb = RustBuild()
 508         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 509         >>> rb.get_toml("key2")
 510         'value2'
 511
 512         If the key does not exists, the result is None:
 513
 514         >>> rb.get_toml("key3") is None
 515         True
 516
 517         Optionally also matches the section the key appears in
 518
 519         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 520         >>> rb.get_toml('key', 'a')
 521         'value1'
 522         >>> rb.get_toml('key', 'b')
 523         'value2'
 524         >>> rb.get_toml('key', 'c') is None
 525         True
 526
 527         >>> rb.config_toml = 'key1 = true'
 528         >>> rb.get_toml("key1")
 529         'true'
 530         """
 531
 532         cur_section = None
 533         for line in self.config_toml.splitlines():
 534             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 535             if section_match is not None:
 536                 cur_section = section_match.group(1)
 537
 538             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 539             if match is not None:
 540                 value = match.group(1)
 541                 if section is None or section == cur_section:
 542                     return self.get_string(value) or value.strip()
 543         return None
 544
 545     def cargo(self):
 546         """Return config path for cargo"""
 547         return self.program_config('cargo')
 548
 549     def rustc(self):
 550         """Return config path for rustc"""
 551         return self.program_config('rustc')
 552
 553     def program_config(self, program):
 554         """Return config path for the given program
 555
 556         >>> rb = RustBuild()
 557         >>> rb.config_toml = 'rustc = "rustc"\\n'
 558         >>> rb.program_config('rustc')
 559         'rustc'
 560         >>> rb.config_toml = ''
 561         >>> cargo_path = rb.program_config('cargo')
 562         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 563         ... "bin", "cargo")
 564         True
 565         """
 566         config = self.get_toml(program)
 567         if config:
 568             return os.path.expanduser(config)
 569         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 570             program, self.exe_suffix()))
 571
 572     @staticmethod
 573     def get_string(line):
 574         """Return the value between double quotes
 575
 576         >>> RustBuild.get_string('    "devel"   ')
 577         'devel'
 578         >>> RustBuild.get_string("    'devel'   ")
 579         'devel'
 580         >>> RustBuild.get_string('devel') is None
 581         True
 582         >>> RustBuild.get_string('    "devel   ')
 583         ''
 584         """
 585         start = line.find('"')
 586         if start != -1:
 587             end = start + 1 + line[start + 1:].find('"')
 588             return line[start + 1:end]
 589         start = line.find('\'')
 590         if start != -1:
 591             end = start + 1 + line[start + 1:].find('\'')
 592             return line[start + 1:end]
 593         return None
 594
 595     @staticmethod
 596     def exe_suffix():
 597         """Return a suffix for executables"""
 598         if sys.platform == 'win32':
 599             return '.exe'
 600         return ''
 601
 602     def bootstrap_binary(self):
 603         """Return the path of the bootstrap binary
 604
 605         >>> rb = RustBuild()
 606         >>> rb.build_dir = "build"
 607         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 608         ... "debug", "bootstrap")
 609         True
 610         """
 611         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 612
 613     def build_bootstrap(self):
 614         """Build bootstrap"""
 615         build_dir = os.path.join(self.build_dir, "bootstrap")
 616         if self.clean and os.path.exists(build_dir):
 617             shutil.rmtree(build_dir)
 618         env = os.environ.copy()
 619         env["RUSTC_BOOTSTRAP"] = '1'
 620         env["CARGO_TARGET_DIR"] = build_dir
 621         env["RUSTC"] = self.rustc()
 622         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 623             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 624             if "LD_LIBRARY_PATH" in env else ""
 625         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 626             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 627             if "DYLD_LIBRARY_PATH" in env else ""
 628         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 629             (os.pathsep + env["LIBRARY_PATH"]) \
 630             if "LIBRARY_PATH" in env else ""
 631         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 632
 633         build_section = "target.{}".format(self.build_triple())
 634         target_features = []
 635         if self.get_toml("crt-static", build_section) == "true":
 636             target_features += ["+crt-static"]
 637         elif self.get_toml("crt-static", build_section) == "false":
 638             target_features += ["-crt-static"]
 639         if target_features:
 640             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 641         target_linker = self.get_toml("linker", build_section)
 642         if target_linker is not None:
 643             env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
 644         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes "
 645         if self.get_toml("deny-warnings", "rust") != "false":
 646             env["RUSTFLAGS"] += "-Dwarnings "
 647
 648         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 649             os.pathsep + env["PATH"]
 650         if not os.path.isfile(self.cargo()):
 651             raise Exception("no cargo executable found at `{}`".format(
 652                 self.cargo()))
 653         args = [self.cargo(), "build", "--manifest-path",
 654                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 655         for _ in range(1, self.verbose):
 656             args.append("--verbose")
 657         if self.use_locked_deps:
 658             args.append("--locked")
 659         if self.use_vendored_sources:
 660             args.append("--frozen")
 661         run(args, env=env, verbose=self.verbose)
 662
 663     def build_triple(self):
 664         """Build triple as in LLVM"""
 665         config = self.get_toml('build')
 666         if config:
 667             return config
 668         return default_build_triple()
 669
 670     def check_submodule(self, module, slow_submodules):
 671         if not slow_submodules:
 672             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 673                                            cwd=os.path.join(self.rust_root, module),
 674                                            stdout=subprocess.PIPE)
 675             return checked_out
 676         else:
 677             return None
 678
 679     def update_submodule(self, module, checked_out, recorded_submodules):
 680         module_path = os.path.join(self.rust_root, module)
 681
 682         if checked_out is not None:
 683             default_encoding = sys.getdefaultencoding()
 684             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 685             if recorded_submodules[module] == checked_out:
 686                 return
 687
 688         print("Updating submodule", module)
 689
 690         run(["git", "submodule", "-q", "sync", module],
 691             cwd=self.rust_root, verbose=self.verbose)
 692         try:
 693             run(["git", "submodule", "update",
 694                  "--init", "--recursive", "--progress", module],
 695                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 696         except RuntimeError:
 697             # Some versions of git don't support --progress.
 698             run(["git", "submodule", "update",
 699                  "--init", "--recursive", module],
 700                 cwd=self.rust_root, verbose=self.verbose)
 701         run(["git", "reset", "-q", "--hard"],
 702             cwd=module_path, verbose=self.verbose)
 703         run(["git", "clean", "-qdfx"],
 704             cwd=module_path, verbose=self.verbose)
 705
 706     def update_submodules(self):
 707         """Update submodules"""
 708         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 709                 self.get_toml('submodules') == "false":
 710             return
 711
 712         # check the existence of 'git' command
 713         try:
 714             subprocess.check_output(['git', '--version'])
 715         except (subprocess.CalledProcessError, OSError):
 716             print("error: `git` is not found, please make sure it's installed and in the path.")
 717             sys.exit(1)
 718
 719         slow_submodules = self.get_toml('fast-submodules') == "false"
 720         start_time = time()
 721         if slow_submodules:
 722             print('Unconditionally updating all submodules')
 723         else:
 724             print('Updating only changed submodules')
 725         default_encoding = sys.getdefaultencoding()
 726         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 727             ["git", "config", "--file",
 728              os.path.join(self.rust_root, ".gitmodules"),
 729              "--get-regexp", "path"]
 730         ).decode(default_encoding).splitlines()]
 731         filtered_submodules = []
 732         submodules_names = []
 733         for module in submodules:
 734             if module.endswith("llvm-project"):
 735                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 736                     continue
 737             check = self.check_submodule(module, slow_submodules)
 738             filtered_submodules.append((module, check))
 739             submodules_names.append(module)
 740         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 741                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 742         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 743         recorded_submodules = {}
 744         for data in recorded:
 745             data = data.split()
 746             recorded_submodules[data[3]] = data[2]
 747         for module in filtered_submodules:
 748             self.update_submodule(module[0], module[1], recorded_submodules)
 749         print("Submodules updated in %.2f seconds" % (time() - start_time))
 750
 751     def set_normal_environment(self):
 752         """Set download URL for normal environment"""
 753         if 'RUSTUP_DIST_SERVER' in os.environ:
 754             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 755         else:
 756             self._download_url = 'https://static.rust-lang.org'
 757
 758     def set_dev_environment(self):
 759         """Set download URL for development environment"""
 760         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 761             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 762         else:
 763             self._download_url = 'https://dev-static.rust-lang.org'
 764
 765     def check_vendored_status(self):
 766         """Check that vendoring is configured properly"""
 767         vendor_dir = os.path.join(self.rust_root, 'vendor')
 768         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 769             if os.environ.get('USER') != os.environ['SUDO_USER']:
 770                 self.use_vendored_sources = True
 771                 print('info: looks like you are running this command under `sudo`')
 772                 print('      and so in order to preserve your $HOME this will now')
 773                 print('      use vendored sources by default.')
 774                 if not os.path.exists(vendor_dir):
 775                     print('error: vendoring required, but vendor directory does not exist.')
 776                     print('       Run `cargo vendor` without sudo to initialize the '
 777                         'vendor directory.')
 778                     raise Exception("{} not found".format(vendor_dir))
 779
 780         if self.use_vendored_sources:
 781             if not os.path.exists('.cargo'):
 782                 os.makedirs('.cargo')
 783             with output('.cargo/config') as cargo_config:
 784                 cargo_config.write(
 785                     "[source.crates-io]\n"
 786                     "replace-with = 'vendored-sources'\n"
 787                     "registry = 'https://example.com'\n"
 788                     "\n"
 789                     "[source.vendored-sources]\n"
 790                     "directory = '{}/vendor'\n"
 791                 .format(self.rust_root))
 792         else:
 793             if os.path.exists('.cargo'):
 794                 shutil.rmtree('.cargo')
 795
 796     def ensure_vendored(self):
 797         """Ensure that the vendored sources are available if needed"""
 798         vendor_dir = os.path.join(self.rust_root, 'vendor')
 799         # Note that this does not handle updating the vendored dependencies if
 800         # the rust git repository is updated. Normal development usually does
 801         # not use vendoring, so hopefully this isn't too much of a problem.
 802         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 803             run([self.cargo(), "vendor"],
 804                 verbose=self.verbose, cwd=self.rust_root)
 805
 806
 807 def bootstrap(help_triggered):
 808     """Configure, fetch, build and run the initial bootstrap"""
 809
 810     # If the user is asking for help, let them know that the whole download-and-build
 811     # process has to happen before anything is printed out.
 812     if help_triggered:
 813         print("info: Downloading and building bootstrap before processing --help")
 814         print("      command. See src/bootstrap/README.md for help with common")
 815         print("      commands.")
 816
 817     parser = argparse.ArgumentParser(description='Build rust')
 818     parser.add_argument('--config')
 819     parser.add_argument('--build')
 820     parser.add_argument('--src')
 821     parser.add_argument('--clean', action='store_true')
 822     parser.add_argument('-v', '--verbose', action='count', default=0)
 823
 824     args = [a for a in sys.argv if a != '-h' and a != '--help']
 825     args, _ = parser.parse_known_args(args)
 826
 827     # Configure initial bootstrap
 828     build = RustBuild()
 829     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 830     build.verbose = args.verbose
 831     build.clean = args.clean
 832
 833     try:
 834         with open(args.config or 'config.toml') as config:
 835             build.config_toml = config.read()
 836     except (OSError, IOError):
 837         pass
 838
 839     config_verbose = build.get_toml('verbose', 'build')
 840     if config_verbose is not None:
 841         build.verbose = max(build.verbose, int(config_verbose))
 842
 843     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 844
 845     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 846
 847     build.check_vendored_status()
 848
 849     data = stage0_data(build.rust_root)
 850     build.date = data['date']
 851     build.rustc_channel = data['rustc']
 852     build.cargo_channel = data['cargo']
 853
 854     if 'dev' in data:
 855         build.set_dev_environment()
 856     else:
 857         build.set_normal_environment()
 858
 859     build.update_submodules()
 860
 861     # Fetch/build the bootstrap
 862     build.build = args.build or build.build_triple()
 863     build.download_stage0()
 864     sys.stdout.flush()
 865     build.ensure_vendored()
 866     build.build_bootstrap()
 867     sys.stdout.flush()
 868
 869     # Run the bootstrap
 870     args = [build.bootstrap_binary()]
 871     args.extend(sys.argv[1:])
 872     env = os.environ.copy()
 873     env["BUILD"] = build.build
 874     env["SRC"] = build.rust_root
 875     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 876     env["BOOTSTRAP_PYTHON"] = sys.executable
 877     env["BUILD_DIR"] = build.build_dir
 878     env["RUSTC_BOOTSTRAP"] = '1'
 879     env["CARGO"] = build.cargo()
 880     env["RUSTC"] = build.rustc()
 881     run(args, env=env, verbose=build.verbose)
 882
 883
 884 def main():
 885     """Entry point for the bootstrap process"""
 886     start_time = time()
 887
 888     # x.py help <cmd> ...
 889     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 890         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 891
 892     help_triggered = (
 893         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 894     try:
 895         bootstrap(help_triggered)
 896         if not help_triggered:
 897             print("Build completed successfully in {}".format(
 898                 format_build_time(time() - start_time)))
 899     except (SystemExit, KeyboardInterrupt) as error:
 900         if hasattr(error, 'code') and isinstance(error.code, int):
 901             exit_code = error.code
 902         else:
 903             exit_code = 1
 904             print(error)
 905         if not help_triggered:
 906             print("Build completed unsuccessfully in {}".format(
 907                 format_build_time(time() - start_time)))
 908         sys.exit(exit_code)
 909
 910
 911 if __name__ == '__main__':
 912     main()