src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = ''
 324         self.rustc_channel = ''
 325         self.build = ''
 326         self.build_dir = os.path.join(os.getcwd(), "build")
 327         self.clean = False
 328         self.config_toml = ''
 329         self.rust_root = ''
 330         self.use_locked_deps = ''
 331         self.use_vendored_sources = ''
 332         self.verbose = False
 333
 334
 335     def download_stage0(self):
 336         """Fetch the build system for Rust, written in Rust
 337
 338         This method will build a cache directory, then it will fetch the
 339         tarball which has the stage0 compiler used to then bootstrap the Rust
 340         compiler itself.
 341
 342         Each downloaded tarball is extracted, after that, the script
 343         will move all the content to the right place.
 344         """
 345         rustc_channel = self.rustc_channel
 346         cargo_channel = self.cargo_channel
 347
 348         def support_xz():
 349             try:
 350                 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
 351                     temp_path = temp_file.name
 352                 with tarfile.open(temp_path, "w:xz") as tar:
 353                     pass
 354                 return True
 355             except tarfile.CompressionError:
 356                 return False
 357
 358         if self.rustc().startswith(self.bin_root()) and \
 359                 (not os.path.exists(self.rustc()) or
 360                  self.program_out_of_date(self.rustc_stamp())):
 361             if os.path.exists(self.bin_root()):
 362                 shutil.rmtree(self.bin_root())
 363             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 364             filename = "rust-std-{}-{}{}".format(
 365                 rustc_channel, self.build, tarball_suffix)
 366             pattern = "rust-std-{}".format(self.build)
 367             self._download_stage0_helper(filename, pattern, tarball_suffix)
 368
 369             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 370                                               tarball_suffix)
 371             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 372             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 373             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 374             with output(self.rustc_stamp()) as rust_stamp:
 375                 rust_stamp.write(self.date)
 376
 377             # This is required so that we don't mix incompatible MinGW
 378             # libraries/binaries that are included in rust-std with
 379             # the system MinGW ones.
 380             if "pc-windows-gnu" in self.build:
 381                 filename = "rust-mingw-{}-{}{}".format(
 382                     rustc_channel, self.build, tarball_suffix)
 383                 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
 384
 385         if self.cargo().startswith(self.bin_root()) and \
 386                 (not os.path.exists(self.cargo()) or
 387                  self.program_out_of_date(self.cargo_stamp())):
 388             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 389             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 390                                               tarball_suffix)
 391             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 392             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 393             with output(self.cargo_stamp()) as cargo_stamp:
 394                 cargo_stamp.write(self.date)
 395
 396     def _download_stage0_helper(self, filename, pattern, tarball_suffix):
 397         cache_dst = os.path.join(self.build_dir, "cache")
 398         rustc_cache = os.path.join(cache_dst, self.date)
 399         if not os.path.exists(rustc_cache):
 400             os.makedirs(rustc_cache)
 401
 402         url = "{}/dist/{}".format(self._download_url, self.date)
 403         tarball = os.path.join(rustc_cache, filename)
 404         if not os.path.exists(tarball):
 405             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 406         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 407
 408     @staticmethod
 409     def fix_executable(fname):
 410         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 411
 412         This method is only required on NixOS and uses the PatchELF utility to
 413         change the dynamic linker of ELF executables.
 414
 415         Please see https://nixos.org/patchelf.html for more information
 416         """
 417         default_encoding = sys.getdefaultencoding()
 418         try:
 419             ostype = subprocess.check_output(
 420                 ['uname', '-s']).strip().decode(default_encoding)
 421         except subprocess.CalledProcessError:
 422             return
 423         except OSError as reason:
 424             if getattr(reason, 'winerror', None) is not None:
 425                 return
 426             raise reason
 427
 428         if ostype != "Linux":
 429             return
 430
 431         if not os.path.exists("/etc/NIXOS"):
 432             return
 433         if os.path.exists("/lib"):
 434             return
 435
 436         # At this point we're pretty sure the user is running NixOS
 437         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 438         print(nix_os_msg, fname)
 439
 440         try:
 441             interpreter = subprocess.check_output(
 442                 ["patchelf", "--print-interpreter", fname])
 443             interpreter = interpreter.strip().decode(default_encoding)
 444         except subprocess.CalledProcessError as reason:
 445             print("warning: failed to call patchelf:", reason)
 446             return
 447
 448         loader = interpreter.split("/")[-1]
 449
 450         try:
 451             ldd_output = subprocess.check_output(
 452                 ['ldd', '/run/current-system/sw/bin/sh'])
 453             ldd_output = ldd_output.strip().decode(default_encoding)
 454         except subprocess.CalledProcessError as reason:
 455             print("warning: unable to call ldd:", reason)
 456             return
 457
 458         for line in ldd_output.splitlines():
 459             libname = line.split()[0]
 460             if libname.endswith(loader):
 461                 loader_path = libname[:len(libname) - len(loader)]
 462                 break
 463         else:
 464             print("warning: unable to find the path to the dynamic linker")
 465             return
 466
 467         correct_interpreter = loader_path + loader
 468
 469         try:
 470             subprocess.check_output(
 471                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 472         except subprocess.CalledProcessError as reason:
 473             print("warning: failed to call patchelf:", reason)
 474             return
 475
 476     def rustc_stamp(self):
 477         """Return the path for .rustc-stamp
 478
 479         >>> rb = RustBuild()
 480         >>> rb.build_dir = "build"
 481         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 482         True
 483         """
 484         return os.path.join(self.bin_root(), '.rustc-stamp')
 485
 486     def cargo_stamp(self):
 487         """Return the path for .cargo-stamp
 488
 489         >>> rb = RustBuild()
 490         >>> rb.build_dir = "build"
 491         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 492         True
 493         """
 494         return os.path.join(self.bin_root(), '.cargo-stamp')
 495
 496     def program_out_of_date(self, stamp_path):
 497         """Check if the given program stamp is out of date"""
 498         if not os.path.exists(stamp_path) or self.clean:
 499             return True
 500         with open(stamp_path, 'r') as stamp:
 501             return self.date != stamp.read()
 502
 503     def bin_root(self):
 504         """Return the binary root directory
 505
 506         >>> rb = RustBuild()
 507         >>> rb.build_dir = "build"
 508         >>> rb.bin_root() == os.path.join("build", "stage0")
 509         True
 510
 511         When the 'build' property is given should be a nested directory:
 512
 513         >>> rb.build = "devel"
 514         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 515         True
 516         """
 517         return os.path.join(self.build_dir, self.build, "stage0")
 518
 519     def get_toml(self, key, section=None):
 520         """Returns the value of the given key in config.toml, otherwise returns None
 521
 522         >>> rb = RustBuild()
 523         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 524         >>> rb.get_toml("key2")
 525         'value2'
 526
 527         If the key does not exists, the result is None:
 528
 529         >>> rb.get_toml("key3") is None
 530         True
 531
 532         Optionally also matches the section the key appears in
 533
 534         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 535         >>> rb.get_toml('key', 'a')
 536         'value1'
 537         >>> rb.get_toml('key', 'b')
 538         'value2'
 539         >>> rb.get_toml('key', 'c') is None
 540         True
 541
 542         >>> rb.config_toml = 'key1 = true'
 543         >>> rb.get_toml("key1")
 544         'true'
 545         """
 546
 547         cur_section = None
 548         for line in self.config_toml.splitlines():
 549             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 550             if section_match is not None:
 551                 cur_section = section_match.group(1)
 552
 553             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 554             if match is not None:
 555                 value = match.group(1)
 556                 if section is None or section == cur_section:
 557                     return self.get_string(value) or value.strip()
 558         return None
 559
 560     def cargo(self):
 561         """Return config path for cargo"""
 562         return self.program_config('cargo')
 563
 564     def rustc(self):
 565         """Return config path for rustc"""
 566         return self.program_config('rustc')
 567
 568     def program_config(self, program):
 569         """Return config path for the given program
 570
 571         >>> rb = RustBuild()
 572         >>> rb.config_toml = 'rustc = "rustc"\\n'
 573         >>> rb.program_config('rustc')
 574         'rustc'
 575         >>> rb.config_toml = ''
 576         >>> cargo_path = rb.program_config('cargo')
 577         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 578         ... "bin", "cargo")
 579         True
 580         """
 581         config = self.get_toml(program)
 582         if config:
 583             return os.path.expanduser(config)
 584         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 585             program, self.exe_suffix()))
 586
 587     @staticmethod
 588     def get_string(line):
 589         """Return the value between double quotes
 590
 591         >>> RustBuild.get_string('    "devel"   ')
 592         'devel'
 593         >>> RustBuild.get_string("    'devel'   ")
 594         'devel'
 595         >>> RustBuild.get_string('devel') is None
 596         True
 597         >>> RustBuild.get_string('    "devel   ')
 598         ''
 599         """
 600         start = line.find('"')
 601         if start != -1:
 602             end = start + 1 + line[start + 1:].find('"')
 603             return line[start + 1:end]
 604         start = line.find('\'')
 605         if start != -1:
 606             end = start + 1 + line[start + 1:].find('\'')
 607             return line[start + 1:end]
 608         return None
 609
 610     @staticmethod
 611     def exe_suffix():
 612         """Return a suffix for executables"""
 613         if sys.platform == 'win32':
 614             return '.exe'
 615         return ''
 616
 617     def bootstrap_binary(self):
 618         """Return the path of the bootstrap binary
 619
 620         >>> rb = RustBuild()
 621         >>> rb.build_dir = "build"
 622         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 623         ... "debug", "bootstrap")
 624         True
 625         """
 626         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 627
 628     def build_bootstrap(self):
 629         """Build bootstrap"""
 630         build_dir = os.path.join(self.build_dir, "bootstrap")
 631         if self.clean and os.path.exists(build_dir):
 632             shutil.rmtree(build_dir)
 633         env = os.environ.copy()
 634         env["RUSTC_BOOTSTRAP"] = '1'
 635         env["CARGO_TARGET_DIR"] = build_dir
 636         env["RUSTC"] = self.rustc()
 637         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 638             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 639             if "LD_LIBRARY_PATH" in env else ""
 640         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 641             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 642             if "DYLD_LIBRARY_PATH" in env else ""
 643         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 644             (os.pathsep + env["LIBRARY_PATH"]) \
 645             if "LIBRARY_PATH" in env else ""
 646         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 647
 648         build_section = "target.{}".format(self.build_triple())
 649         target_features = []
 650         if self.get_toml("crt-static", build_section) == "true":
 651             target_features += ["+crt-static"]
 652         elif self.get_toml("crt-static", build_section) == "false":
 653             target_features += ["-crt-static"]
 654         if target_features:
 655             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 656         target_linker = self.get_toml("linker", build_section)
 657         if target_linker is not None:
 658             env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
 659         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes "
 660         if self.get_toml("deny-warnings", "rust") != "false":
 661             env["RUSTFLAGS"] += "-Dwarnings "
 662
 663         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 664             os.pathsep + env["PATH"]
 665         if not os.path.isfile(self.cargo()):
 666             raise Exception("no cargo executable found at `{}`".format(
 667                 self.cargo()))
 668         args = [self.cargo(), "build", "--manifest-path",
 669                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 670         for _ in range(1, self.verbose):
 671             args.append("--verbose")
 672         if self.use_locked_deps:
 673             args.append("--locked")
 674         if self.use_vendored_sources:
 675             args.append("--frozen")
 676         run(args, env=env, verbose=self.verbose)
 677
 678     def build_triple(self):
 679         """Build triple as in LLVM"""
 680         config = self.get_toml('build')
 681         if config:
 682             return config
 683         return default_build_triple()
 684
 685     def check_submodule(self, module, slow_submodules):
 686         if not slow_submodules:
 687             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 688                                            cwd=os.path.join(self.rust_root, module),
 689                                            stdout=subprocess.PIPE)
 690             return checked_out
 691         else:
 692             return None
 693
 694     def update_submodule(self, module, checked_out, recorded_submodules):
 695         module_path = os.path.join(self.rust_root, module)
 696
 697         if checked_out is not None:
 698             default_encoding = sys.getdefaultencoding()
 699             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 700             if recorded_submodules[module] == checked_out:
 701                 return
 702
 703         print("Updating submodule", module)
 704
 705         run(["git", "submodule", "-q", "sync", module],
 706             cwd=self.rust_root, verbose=self.verbose)
 707         try:
 708             run(["git", "submodule", "update",
 709                  "--init", "--recursive", "--progress", module],
 710                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 711         except RuntimeError:
 712             # Some versions of git don't support --progress.
 713             run(["git", "submodule", "update",
 714                  "--init", "--recursive", module],
 715                 cwd=self.rust_root, verbose=self.verbose)
 716         run(["git", "reset", "-q", "--hard"],
 717             cwd=module_path, verbose=self.verbose)
 718         run(["git", "clean", "-qdfx"],
 719             cwd=module_path, verbose=self.verbose)
 720
 721     def update_submodules(self):
 722         """Update submodules"""
 723         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 724                 self.get_toml('submodules') == "false":
 725             return
 726
 727         # check the existence of 'git' command
 728         try:
 729             subprocess.check_output(['git', '--version'])
 730         except (subprocess.CalledProcessError, OSError):
 731             print("error: `git` is not found, please make sure it's installed and in the path.")
 732             sys.exit(1)
 733
 734         slow_submodules = self.get_toml('fast-submodules') == "false"
 735         start_time = time()
 736         if slow_submodules:
 737             print('Unconditionally updating all submodules')
 738         else:
 739             print('Updating only changed submodules')
 740         default_encoding = sys.getdefaultencoding()
 741         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 742             ["git", "config", "--file",
 743              os.path.join(self.rust_root, ".gitmodules"),
 744              "--get-regexp", "path"]
 745         ).decode(default_encoding).splitlines()]
 746         filtered_submodules = []
 747         submodules_names = []
 748         for module in submodules:
 749             if module.endswith("llvm-project"):
 750                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 751                     continue
 752             check = self.check_submodule(module, slow_submodules)
 753             filtered_submodules.append((module, check))
 754             submodules_names.append(module)
 755         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 756                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 757         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 758         recorded_submodules = {}
 759         for data in recorded:
 760             data = data.split()
 761             recorded_submodules[data[3]] = data[2]
 762         for module in filtered_submodules:
 763             self.update_submodule(module[0], module[1], recorded_submodules)
 764         print("Submodules updated in %.2f seconds" % (time() - start_time))
 765
 766     def set_normal_environment(self):
 767         """Set download URL for normal environment"""
 768         if 'RUSTUP_DIST_SERVER' in os.environ:
 769             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 770         else:
 771             self._download_url = 'https://static.rust-lang.org'
 772
 773     def set_dev_environment(self):
 774         """Set download URL for development environment"""
 775         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 776             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 777         else:
 778             self._download_url = 'https://dev-static.rust-lang.org'
 779
 780     def check_vendored_status(self):
 781         """Check that vendoring is configured properly"""
 782         vendor_dir = os.path.join(self.rust_root, 'vendor')
 783         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 784             if os.environ.get('USER') != os.environ['SUDO_USER']:
 785                 self.use_vendored_sources = True
 786                 print('info: looks like you are running this command under `sudo`')
 787                 print('      and so in order to preserve your $HOME this will now')
 788                 print('      use vendored sources by default.')
 789                 if not os.path.exists(vendor_dir):
 790                     print('error: vendoring required, but vendor directory does not exist.')
 791                     print('       Run `cargo vendor` without sudo to initialize the '
 792                         'vendor directory.')
 793                     raise Exception("{} not found".format(vendor_dir))
 794
 795         if self.use_vendored_sources:
 796             if not os.path.exists('.cargo'):
 797                 os.makedirs('.cargo')
 798             with output('.cargo/config') as cargo_config:
 799                 cargo_config.write(
 800                     "[source.crates-io]\n"
 801                     "replace-with = 'vendored-sources'\n"
 802                     "registry = 'https://example.com'\n"
 803                     "\n"
 804                     "[source.vendored-sources]\n"
 805                     "directory = '{}/vendor'\n"
 806                 .format(self.rust_root))
 807         else:
 808             if os.path.exists('.cargo'):
 809                 shutil.rmtree('.cargo')
 810
 811     def ensure_vendored(self):
 812         """Ensure that the vendored sources are available if needed"""
 813         vendor_dir = os.path.join(self.rust_root, 'vendor')
 814         # Note that this does not handle updating the vendored dependencies if
 815         # the rust git repository is updated. Normal development usually does
 816         # not use vendoring, so hopefully this isn't too much of a problem.
 817         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 818             run([self.cargo(), "vendor"],
 819                 verbose=self.verbose, cwd=self.rust_root)
 820
 821
 822 def bootstrap(help_triggered):
 823     """Configure, fetch, build and run the initial bootstrap"""
 824
 825     # If the user is asking for help, let them know that the whole download-and-build
 826     # process has to happen before anything is printed out.
 827     if help_triggered:
 828         print("info: Downloading and building bootstrap before processing --help")
 829         print("      command. See src/bootstrap/README.md for help with common")
 830         print("      commands.")
 831
 832     parser = argparse.ArgumentParser(description='Build rust')
 833     parser.add_argument('--config')
 834     parser.add_argument('--build')
 835     parser.add_argument('--src')
 836     parser.add_argument('--clean', action='store_true')
 837     parser.add_argument('-v', '--verbose', action='count', default=0)
 838
 839     args = [a for a in sys.argv if a != '-h' and a != '--help']
 840     args, _ = parser.parse_known_args(args)
 841
 842     # Configure initial bootstrap
 843     build = RustBuild()
 844     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 845     build.verbose = args.verbose
 846     build.clean = args.clean
 847
 848     try:
 849         with open(args.config or 'config.toml') as config:
 850             build.config_toml = config.read()
 851     except (OSError, IOError):
 852         pass
 853
 854     config_verbose = build.get_toml('verbose', 'build')
 855     if config_verbose is not None:
 856         build.verbose = max(build.verbose, int(config_verbose))
 857
 858     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 859
 860     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 861
 862     build.check_vendored_status()
 863
 864     data = stage0_data(build.rust_root)
 865     build.date = data['date']
 866     build.rustc_channel = data['rustc']
 867     build.cargo_channel = data['cargo']
 868
 869     if 'dev' in data:
 870         build.set_dev_environment()
 871     else:
 872         build.set_normal_environment()
 873
 874     build.update_submodules()
 875
 876     # Fetch/build the bootstrap
 877     build.build = args.build or build.build_triple()
 878     build.download_stage0()
 879     sys.stdout.flush()
 880     build.ensure_vendored()
 881     build.build_bootstrap()
 882     sys.stdout.flush()
 883
 884     # Run the bootstrap
 885     args = [build.bootstrap_binary()]
 886     args.extend(sys.argv[1:])
 887     env = os.environ.copy()
 888     env["BUILD"] = build.build
 889     env["SRC"] = build.rust_root
 890     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 891     env["BOOTSTRAP_PYTHON"] = sys.executable
 892     env["BUILD_DIR"] = build.build_dir
 893     env["RUSTC_BOOTSTRAP"] = '1'
 894     env["CARGO"] = build.cargo()
 895     env["RUSTC"] = build.rustc()
 896     run(args, env=env, verbose=build.verbose)
 897
 898
 899 def main():
 900     """Entry point for the bootstrap process"""
 901     start_time = time()
 902
 903     # x.py help <cmd> ...
 904     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 905         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 906
 907     help_triggered = (
 908         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 909     try:
 910         bootstrap(help_triggered)
 911         if not help_triggered:
 912             print("Build completed successfully in {}".format(
 913                 format_build_time(time() - start_time)))
 914     except (SystemExit, KeyboardInterrupt) as error:
 915         if hasattr(error, 'code') and isinstance(error.code, int):
 916             exit_code = error.code
 917         else:
 918             exit_code = 1
 919             print(error)
 920         if not help_triggered:
 921             print("Build completed unsuccessfully in {}".format(
 922                 format_build_time(time() - start_time)))
 923         sys.exit(exit_code)
 924
 925
 926 if __name__ == '__main__':
 927     main()