src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = ''
 324         self.rustc_channel = ''
 325         self.build = ''
 326         self.build_dir = os.path.join(os.getcwd(), "build")
 327         self.clean = False
 328         self.config_toml = ''
 329         self.rust_root = ''
 330         self.use_locked_deps = ''
 331         self.use_vendored_sources = ''
 332         self.verbose = False
 333
 334
 335     def download_stage0(self):
 336         """Fetch the build system for Rust, written in Rust
 337
 338         This method will build a cache directory, then it will fetch the
 339         tarball which has the stage0 compiler used to then bootstrap the Rust
 340         compiler itself.
 341
 342         Each downloaded tarball is extracted, after that, the script
 343         will move all the content to the right place.
 344         """
 345         rustc_channel = self.rustc_channel
 346         cargo_channel = self.cargo_channel
 347
 348         def support_xz():
 349             try:
 350                 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
 351                     temp_path = temp_file.name
 352                 with tarfile.open(temp_path, "w:xz") as tar:
 353                     pass
 354                 return True
 355             except tarfile.CompressionError:
 356                 return False
 357
 358         if self.rustc().startswith(self.bin_root()) and \
 359                 (not os.path.exists(self.rustc()) or
 360                  self.program_out_of_date(self.rustc_stamp())):
 361             if os.path.exists(self.bin_root()):
 362                 shutil.rmtree(self.bin_root())
 363             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 364             filename = "rust-std-{}-{}{}".format(
 365                 rustc_channel, self.build, tarball_suffix)
 366             pattern = "rust-std-{}".format(self.build)
 367             self._download_stage0_helper(filename, pattern, tarball_suffix)
 368
 369             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 370                                               tarball_suffix)
 371             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 372             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 373             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 374             with output(self.rustc_stamp()) as rust_stamp:
 375                 rust_stamp.write(self.date)
 376
 377             # This is required so that we don't mix incompatible MinGW
 378             # libraries/binaries that are included in rust-std with
 379             # the system MinGW ones.
 380             if "pc-windows-gnu" in self.build:
 381                 filename = "rust-mingw-{}-{}{}".format(
 382                     rustc_channel, self.build, tarball_suffix)
 383                 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
 384
 385         if self.cargo().startswith(self.bin_root()) and \
 386                 (not os.path.exists(self.cargo()) or
 387                  self.program_out_of_date(self.cargo_stamp())):
 388             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 389             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 392             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 393             with output(self.cargo_stamp()) as cargo_stamp:
 394                 cargo_stamp.write(self.date)
 395
 396     def _download_stage0_helper(self, filename, pattern, tarball_suffix):
 397         cache_dst = os.path.join(self.build_dir, "cache")
 398         rustc_cache = os.path.join(cache_dst, self.date)
 399         if not os.path.exists(rustc_cache):
 400             os.makedirs(rustc_cache)
 401
 402         url = "{}/dist/{}".format(self._download_url, self.date)
 403         tarball = os.path.join(rustc_cache, filename)
 404         if not os.path.exists(tarball):
 405             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 406         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 407
 408     @staticmethod
 409     def fix_executable(fname):
 410         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 411
 412         This method is only required on NixOS and uses the PatchELF utility to
 413         change the dynamic linker of ELF executables.
 414
 415         Please see https://nixos.org/patchelf.html for more information
 416         """
 417         default_encoding = sys.getdefaultencoding()
 418         try:
 419             ostype = subprocess.check_output(
 420                 ['uname', '-s']).strip().decode(default_encoding)
 421         except subprocess.CalledProcessError:
 422             return
 423         except OSError as reason:
 424             if getattr(reason, 'winerror', None) is not None:
 425                 return
 426             raise reason
 427
 428         if ostype != "Linux":
 429             return
 430
 431         if not os.path.exists("/etc/NIXOS"):
 432             return
 433         if os.path.exists("/lib"):
 434             return
 435
 436         # At this point we're pretty sure the user is running NixOS
 437         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 438         print(nix_os_msg, fname)
 439
 440         try:
 441             interpreter = subprocess.check_output(
 442                 ["patchelf", "--print-interpreter", fname])
 443             interpreter = interpreter.strip().decode(default_encoding)
 444         except subprocess.CalledProcessError as reason:
 445             print("warning: failed to call patchelf:", reason)
 446             return
 447
 448         loader = interpreter.split("/")[-1]
 449
 450         try:
 451             ldd_output = subprocess.check_output(
 452                 ['ldd', '/run/current-system/sw/bin/sh'])
 453             ldd_output = ldd_output.strip().decode(default_encoding)
 454         except subprocess.CalledProcessError as reason:
 455             print("warning: unable to call ldd:", reason)
 456             return
 457
 458         for line in ldd_output.splitlines():
 459             libname = line.split()[0]
 460             if libname.endswith(loader):
 461                 loader_path = libname[:len(libname) - len(loader)]
 462                 break
 463         else:
 464             print("warning: unable to find the path to the dynamic linker")
 465             return
 466
 467         correct_interpreter = loader_path + loader
 468
 469         try:
 470             subprocess.check_output(
 471                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 472         except subprocess.CalledProcessError as reason:
 473             print("warning: failed to call patchelf:", reason)
 474             return
 475
 476     def rustc_stamp(self):
 477         """Return the path for .rustc-stamp
 478
 479         >>> rb = RustBuild()
 480         >>> rb.build_dir = "build"
 481         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 482         True
 483         """
 484         return os.path.join(self.bin_root(), '.rustc-stamp')
 485
 486     def cargo_stamp(self):
 487         """Return the path for .cargo-stamp
 488
 489         >>> rb = RustBuild()
 490         >>> rb.build_dir = "build"
 491         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 492         True
 493         """
 494         return os.path.join(self.bin_root(), '.cargo-stamp')
 495
 496     def program_out_of_date(self, stamp_path):
 497         """Check if the given program stamp is out of date"""
 498         if not os.path.exists(stamp_path) or self.clean:
 499             return True
 500         with open(stamp_path, 'r') as stamp:
 501             return self.date != stamp.read()
 502
 503     def bin_root(self):
 504         """Return the binary root directory
 505
 506         >>> rb = RustBuild()
 507         >>> rb.build_dir = "build"
 508         >>> rb.bin_root() == os.path.join("build", "stage0")
 509         True
 510
 511         When the 'build' property is given should be a nested directory:
 512
 513         >>> rb.build = "devel"
 514         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 515         True
 516         """
 517         return os.path.join(self.build_dir, self.build, "stage0")
 518
 519     def get_toml(self, key, section=None):
 520         """Returns the value of the given key in config.toml, otherwise returns None
 521
 522         >>> rb = RustBuild()
 523         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 524         >>> rb.get_toml("key2")
 525         'value2'
 526
 527         If the key does not exists, the result is None:
 528
 529         >>> rb.get_toml("key3") is None
 530         True
 531
 532         Optionally also matches the section the key appears in
 533
 534         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 535         >>> rb.get_toml('key', 'a')
 536         'value1'
 537         >>> rb.get_toml('key', 'b')
 538         'value2'
 539         >>> rb.get_toml('key', 'c') is None
 540         True
 541
 542         >>> rb.config_toml = 'key1 = true'
 543         >>> rb.get_toml("key1")
 544         'true'
 545         """
 546
 547         cur_section = None
 548         for line in self.config_toml.splitlines():
 549             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 550             if section_match is not None:
 551                 cur_section = section_match.group(1)
 552
 553             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 554             if match is not None:
 555                 value = match.group(1)
 556                 if section is None or section == cur_section:
 557                     return self.get_string(value) or value.strip()
 558         return None
 559
 560     def cargo(self):
 561         """Return config path for cargo"""
 562         return self.program_config('cargo')
 563
 564     def rustc(self):
 565         """Return config path for rustc"""
 566         return self.program_config('rustc')
 567
 568     def program_config(self, program):
 569         """Return config path for the given program
 570
 571         >>> rb = RustBuild()
 572         >>> rb.config_toml = 'rustc = "rustc"\\n'
 573         >>> rb.program_config('rustc')
 574         'rustc'
 575         >>> rb.config_toml = ''
 576         >>> cargo_path = rb.program_config('cargo')
 577         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 578         ... "bin", "cargo")
 579         True
 580         """
 581         config = self.get_toml(program)
 582         if config:
 583             return os.path.expanduser(config)
 584         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 585             program, self.exe_suffix()))
 586
 587     @staticmethod
 588     def get_string(line):
 589         """Return the value between double quotes
 590
 591         >>> RustBuild.get_string('    "devel"   ')
 592         'devel'
 593         >>> RustBuild.get_string("    'devel'   ")
 594         'devel'
 595         >>> RustBuild.get_string('devel') is None
 596         True
 597         >>> RustBuild.get_string('    "devel   ')
 598         ''
 599         """
 600         start = line.find('"')
 601         if start != -1:
 602             end = start + 1 + line[start + 1:].find('"')
 603             return line[start + 1:end]
 604         start = line.find('\'')
 605         if start != -1:
 606             end = start + 1 + line[start + 1:].find('\'')
 607             return line[start + 1:end]
 608         return None
 609
 610     @staticmethod
 611     def exe_suffix():
 612         """Return a suffix for executables"""
 613         if sys.platform == 'win32':
 614             return '.exe'
 615         return ''
 616
 617     def bootstrap_binary(self):
 618         """Return the path of the bootstrap binary
 619
 620         >>> rb = RustBuild()
 621         >>> rb.build_dir = "build"
 622         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 623         ... "debug", "bootstrap")
 624         True
 625         """
 626         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 627
 628     def build_bootstrap(self):
 629         """Build bootstrap"""
 630         build_dir = os.path.join(self.build_dir, "bootstrap")
 631         if self.clean and os.path.exists(build_dir):
 632             shutil.rmtree(build_dir)
 633         env = os.environ.copy()
 634         env["RUSTC_BOOTSTRAP"] = '1'
 635         env["CARGO_TARGET_DIR"] = build_dir
 636         env["RUSTC"] = self.rustc()
 637         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 638             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 639             if "LD_LIBRARY_PATH" in env else ""
 640         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 641             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 642             if "DYLD_LIBRARY_PATH" in env else ""
 643         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 644             (os.pathsep + env["LIBRARY_PATH"]) \
 645             if "LIBRARY_PATH" in env else ""
 646         # preserve existing RUSTFLAGS
 647         env.setdefault("RUSTFLAGS", "")
 648         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 649
 650         build_section = "target.{}".format(self.build_triple())
 651         target_features = []
 652         if self.get_toml("crt-static", build_section) == "true":
 653             target_features += ["+crt-static"]
 654         elif self.get_toml("crt-static", build_section) == "false":
 655             target_features += ["-crt-static"]
 656         if target_features:
 657             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 658         target_linker = self.get_toml("linker", build_section)
 659         if target_linker is not None:
 660             env["RUSTFLAGS"] += " -C linker=" + target_linker
 661         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 662         if self.get_toml("deny-warnings", "rust") != "false":
 663             env["RUSTFLAGS"] += " -Dwarnings"
 664
 665         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 666             os.pathsep + env["PATH"]
 667         if not os.path.isfile(self.cargo()):
 668             raise Exception("no cargo executable found at `{}`".format(
 669                 self.cargo()))
 670         args = [self.cargo(), "build", "--manifest-path",
 671                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 672         for _ in range(1, self.verbose):
 673             args.append("--verbose")
 674         if self.use_locked_deps:
 675             args.append("--locked")
 676         if self.use_vendored_sources:
 677             args.append("--frozen")
 678         run(args, env=env, verbose=self.verbose)
 679
 680     def build_triple(self):
 681         """Build triple as in LLVM"""
 682         config = self.get_toml('build')
 683         if config:
 684             return config
 685         return default_build_triple()
 686
 687     def check_submodule(self, module, slow_submodules):
 688         if not slow_submodules:
 689             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 690                                            cwd=os.path.join(self.rust_root, module),
 691                                            stdout=subprocess.PIPE)
 692             return checked_out
 693         else:
 694             return None
 695
 696     def update_submodule(self, module, checked_out, recorded_submodules):
 697         module_path = os.path.join(self.rust_root, module)
 698
 699         if checked_out is not None:
 700             default_encoding = sys.getdefaultencoding()
 701             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 702             if recorded_submodules[module] == checked_out:
 703                 return
 704
 705         print("Updating submodule", module)
 706
 707         run(["git", "submodule", "-q", "sync", module],
 708             cwd=self.rust_root, verbose=self.verbose)
 709         try:
 710             run(["git", "submodule", "update",
 711                  "--init", "--recursive", "--progress", module],
 712                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 713         except RuntimeError:
 714             # Some versions of git don't support --progress.
 715             run(["git", "submodule", "update",
 716                  "--init", "--recursive", module],
 717                 cwd=self.rust_root, verbose=self.verbose)
 718         run(["git", "reset", "-q", "--hard"],
 719             cwd=module_path, verbose=self.verbose)
 720         run(["git", "clean", "-qdfx"],
 721             cwd=module_path, verbose=self.verbose)
 722
 723     def update_submodules(self):
 724         """Update submodules"""
 725         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 726                 self.get_toml('submodules') == "false":
 727             return
 728
 729         # check the existence of 'git' command
 730         try:
 731             subprocess.check_output(['git', '--version'])
 732         except (subprocess.CalledProcessError, OSError):
 733             print("error: `git` is not found, please make sure it's installed and in the path.")
 734             sys.exit(1)
 735
 736         slow_submodules = self.get_toml('fast-submodules') == "false"
 737         start_time = time()
 738         if slow_submodules:
 739             print('Unconditionally updating all submodules')
 740         else:
 741             print('Updating only changed submodules')
 742         default_encoding = sys.getdefaultencoding()
 743         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 744             ["git", "config", "--file",
 745              os.path.join(self.rust_root, ".gitmodules"),
 746              "--get-regexp", "path"]
 747         ).decode(default_encoding).splitlines()]
 748         filtered_submodules = []
 749         submodules_names = []
 750         for module in submodules:
 751             if module.endswith("llvm-project"):
 752                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 753                     continue
 754             check = self.check_submodule(module, slow_submodules)
 755             filtered_submodules.append((module, check))
 756             submodules_names.append(module)
 757         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 758                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 759         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 760         recorded_submodules = {}
 761         for data in recorded:
 762             data = data.split()
 763             recorded_submodules[data[3]] = data[2]
 764         for module in filtered_submodules:
 765             self.update_submodule(module[0], module[1], recorded_submodules)
 766         print("Submodules updated in %.2f seconds" % (time() - start_time))
 767
 768     def set_normal_environment(self):
 769         """Set download URL for normal environment"""
 770         if 'RUSTUP_DIST_SERVER' in os.environ:
 771             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 772         else:
 773             self._download_url = 'https://static.rust-lang.org'
 774
 775     def set_dev_environment(self):
 776         """Set download URL for development environment"""
 777         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 778             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 779         else:
 780             self._download_url = 'https://dev-static.rust-lang.org'
 781
 782     def check_vendored_status(self):
 783         """Check that vendoring is configured properly"""
 784         vendor_dir = os.path.join(self.rust_root, 'vendor')
 785         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 786             if os.environ.get('USER') != os.environ['SUDO_USER']:
 787                 self.use_vendored_sources = True
 788                 print('info: looks like you are running this command under `sudo`')
 789                 print('      and so in order to preserve your $HOME this will now')
 790                 print('      use vendored sources by default.')
 791                 if not os.path.exists(vendor_dir):
 792                     print('error: vendoring required, but vendor directory does not exist.')
 793                     print('       Run `cargo vendor` without sudo to initialize the '
 794                         'vendor directory.')
 795                     raise Exception("{} not found".format(vendor_dir))
 796
 797         if self.use_vendored_sources:
 798             if not os.path.exists('.cargo'):
 799                 os.makedirs('.cargo')
 800             with output('.cargo/config') as cargo_config:
 801                 cargo_config.write(
 802                     "[source.crates-io]\n"
 803                     "replace-with = 'vendored-sources'\n"
 804                     "registry = 'https://example.com'\n"
 805                     "\n"
 806                     "[source.vendored-sources]\n"
 807                     "directory = '{}/vendor'\n"
 808                 .format(self.rust_root))
 809         else:
 810             if os.path.exists('.cargo'):
 811                 shutil.rmtree('.cargo')
 812
 813     def ensure_vendored(self):
 814         """Ensure that the vendored sources are available if needed"""
 815         vendor_dir = os.path.join(self.rust_root, 'vendor')
 816         # Note that this does not handle updating the vendored dependencies if
 817         # the rust git repository is updated. Normal development usually does
 818         # not use vendoring, so hopefully this isn't too much of a problem.
 819         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 820             run([self.cargo(), "vendor"],
 821                 verbose=self.verbose, cwd=self.rust_root)
 822
 823
 824 def bootstrap(help_triggered):
 825     """Configure, fetch, build and run the initial bootstrap"""
 826
 827     # If the user is asking for help, let them know that the whole download-and-build
 828     # process has to happen before anything is printed out.
 829     if help_triggered:
 830         print("info: Downloading and building bootstrap before processing --help")
 831         print("      command. See src/bootstrap/README.md for help with common")
 832         print("      commands.")
 833
 834     parser = argparse.ArgumentParser(description='Build rust')
 835     parser.add_argument('--config')
 836     parser.add_argument('--build')
 837     parser.add_argument('--src')
 838     parser.add_argument('--clean', action='store_true')
 839     parser.add_argument('-v', '--verbose', action='count', default=0)
 840
 841     args = [a for a in sys.argv if a != '-h' and a != '--help']
 842     args, _ = parser.parse_known_args(args)
 843
 844     # Configure initial bootstrap
 845     build = RustBuild()
 846     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 847     build.verbose = args.verbose
 848     build.clean = args.clean
 849
 850     try:
 851         with open(args.config or 'config.toml') as config:
 852             build.config_toml = config.read()
 853     except (OSError, IOError):
 854         pass
 855
 856     config_verbose = build.get_toml('verbose', 'build')
 857     if config_verbose is not None:
 858         build.verbose = max(build.verbose, int(config_verbose))
 859
 860     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 861
 862     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 863
 864     build.check_vendored_status()
 865
 866     data = stage0_data(build.rust_root)
 867     build.date = data['date']
 868     build.rustc_channel = data['rustc']
 869     build.cargo_channel = data['cargo']
 870
 871     if 'dev' in data:
 872         build.set_dev_environment()
 873     else:
 874         build.set_normal_environment()
 875
 876     build.update_submodules()
 877
 878     # Fetch/build the bootstrap
 879     build.build = args.build or build.build_triple()
 880     build.download_stage0()
 881     sys.stdout.flush()
 882     build.ensure_vendored()
 883     build.build_bootstrap()
 884     sys.stdout.flush()
 885
 886     # Run the bootstrap
 887     args = [build.bootstrap_binary()]
 888     args.extend(sys.argv[1:])
 889     env = os.environ.copy()
 890     env["BUILD"] = build.build
 891     env["SRC"] = build.rust_root
 892     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 893     env["BOOTSTRAP_PYTHON"] = sys.executable
 894     env["BUILD_DIR"] = build.build_dir
 895     env["RUSTC_BOOTSTRAP"] = '1'
 896     env["CARGO"] = build.cargo()
 897     env["RUSTC"] = build.rustc()
 898     run(args, env=env, verbose=build.verbose)
 899
 900
 901 def main():
 902     """Entry point for the bootstrap process"""
 903     start_time = time()
 904
 905     # x.py help <cmd> ...
 906     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 907         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 908
 909     help_triggered = (
 910         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 911     try:
 912         bootstrap(help_triggered)
 913         if not help_triggered:
 914             print("Build completed successfully in {}".format(
 915                 format_build_time(time() - start_time)))
 916     except (SystemExit, KeyboardInterrupt) as error:
 917         if hasattr(error, 'code') and isinstance(error.code, int):
 918             exit_code = error.code
 919         else:
 920             exit_code = 1
 921             print(error)
 922         if not help_triggered:
 923             print("Build completed unsuccessfully in {}".format(
 924                 format_build_time(time() - start_time)))
 925         sys.exit(exit_code)
 926
 927
 928 if __name__ == '__main__':
 929     main()