src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 from __future__ import absolute_import, division, print_function
  12 import argparse
  13 import contextlib
  14 import datetime
  15 import hashlib
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import sys
  21 import tarfile
  22 import tempfile
  23
  24 from time import time
  25
  26
  27 def get(url, path, verbose=False):
  28     suffix = '.sha256'
  29     sha_url = url + suffix
  30     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  31         temp_path = temp_file.name
  32     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  33         sha_path = sha_file.name
  34
  35     try:
  36         download(sha_path, sha_url, False, verbose)
  37         if os.path.exists(path):
  38             if verify(path, sha_path, False):
  39                 if verbose:
  40                     print("using already-download file", path)
  41                 return
  42             else:
  43                 if verbose:
  44                     print("ignoring already-download file",
  45                           path, "due to failed verification")
  46                 os.unlink(path)
  47         download(temp_path, url, True, verbose)
  48         if not verify(temp_path, sha_path, verbose):
  49             raise RuntimeError("failed verification")
  50         if verbose:
  51             print("moving {} to {}".format(temp_path, path))
  52         shutil.move(temp_path, path)
  53     finally:
  54         delete_if_present(sha_path, verbose)
  55         delete_if_present(temp_path, verbose)
  56
  57
  58 def delete_if_present(path, verbose):
  59     """Remove the given file if present"""
  60     if os.path.isfile(path):
  61         if verbose:
  62             print("removing", path)
  63         os.unlink(path)
  64
  65
  66 def download(path, url, probably_big, verbose):
  67     for _ in range(0, 4):
  68         try:
  69             _download(path, url, probably_big, verbose, True)
  70             return
  71         except RuntimeError:
  72             print("\nspurious failure, trying again")
  73     _download(path, url, probably_big, verbose, False)
  74
  75
  76 def _download(path, url, probably_big, verbose, exception):
  77     if probably_big or verbose:
  78         print("downloading {}".format(url))
  79     # see http://serverfault.com/questions/301128/how-to-download
  80     if sys.platform == 'win32':
  81         run(["PowerShell.exe", "/nologo", "-Command",
  82              "(New-Object System.Net.WebClient)"
  83              ".DownloadFile('{}', '{}')".format(url, path)],
  84             verbose=verbose,
  85             exception=exception)
  86     else:
  87         if probably_big or verbose:
  88             option = "-#"
  89         else:
  90             option = "-s"
  91         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
  92             verbose=verbose,
  93             exception=exception)
  94
  95
  96 def verify(path, sha_path, verbose):
  97     """Check if the sha256 sum of the given path is valid"""
  98     if verbose:
  99         print("verifying", path)
 100     with open(path, "rb") as source:
 101         found = hashlib.sha256(source.read()).hexdigest()
 102     with open(sha_path, "r") as sha256sum:
 103         expected = sha256sum.readline().split()[0]
 104     verified = found == expected
 105     if not verified:
 106         print("invalid checksum:\n"
 107               "    found:    {}\n"
 108               "    expected: {}".format(found, expected))
 109     return verified
 110
 111
 112 def unpack(tarball, dst, verbose=False, match=None):
 113     """Unpack the given tarball file"""
 114     print("extracting", tarball)
 115     fname = os.path.basename(tarball).replace(".tar.gz", "")
 116     with contextlib.closing(tarfile.open(tarball)) as tar:
 117         for member in tar.getnames():
 118             if "/" not in member:
 119                 continue
 120             name = member.replace(fname + "/", "", 1)
 121             if match is not None and not name.startswith(match):
 122                 continue
 123             name = name[len(match) + 1:]
 124
 125             dst_path = os.path.join(dst, name)
 126             if verbose:
 127                 print("  extracting", member)
 128             tar.extract(member, dst)
 129             src_path = os.path.join(dst, member)
 130             if os.path.isdir(src_path) and os.path.exists(dst_path):
 131                 continue
 132             shutil.move(src_path, dst_path)
 133     shutil.rmtree(os.path.join(dst, fname))
 134
 135
 136 def run(args, verbose=False, exception=False, **kwargs):
 137     """Run a child program in a new process"""
 138     if verbose:
 139         print("running: " + ' '.join(args))
 140     sys.stdout.flush()
 141     # Use Popen here instead of call() as it apparently allows powershell on
 142     # Windows to not lock up waiting for input presumably.
 143     ret = subprocess.Popen(args, **kwargs)
 144     code = ret.wait()
 145     if code != 0:
 146         err = "failed to run: " + ' '.join(args)
 147         if verbose or exception:
 148             raise RuntimeError(err)
 149         sys.exit(err)
 150
 151
 152 def stage0_data(rust_root):
 153     """Build a dictionary from stage0.txt"""
 154     nightlies = os.path.join(rust_root, "src/stage0.txt")
 155     with open(nightlies, 'r') as nightlies:
 156         lines = [line.rstrip() for line in nightlies
 157                  if not line.startswith("#")]
 158         return dict([line.split(": ", 1) for line in lines if line])
 159
 160
 161 def format_build_time(duration):
 162     """Return a nicer format for build time
 163
 164     >>> format_build_time('300')
 165     '0:05:00'
 166     """
 167     return str(datetime.timedelta(seconds=int(duration)))
 168
 169
 170 def default_build_triple():
 171     """Build triple as in LLVM"""
 172     default_encoding = sys.getdefaultencoding()
 173     try:
 174         ostype = subprocess.check_output(
 175             ['uname', '-s']).strip().decode(default_encoding)
 176         cputype = subprocess.check_output(
 177             ['uname', '-m']).strip().decode(default_encoding)
 178     except (subprocess.CalledProcessError, OSError):
 179         if sys.platform == 'win32':
 180             return 'x86_64-pc-windows-msvc'
 181         err = "uname not found"
 182         sys.exit(err)
 183
 184     # The goal here is to come up with the same triple as LLVM would,
 185     # at least for the subset of platforms we're willing to target.
 186     ostype_mapper = {
 187         'Bitrig': 'unknown-bitrig',
 188         'Darwin': 'apple-darwin',
 189         'DragonFly': 'unknown-dragonfly',
 190         'FreeBSD': 'unknown-freebsd',
 191         'Haiku': 'unknown-haiku',
 192         'NetBSD': 'unknown-netbsd',
 193         'OpenBSD': 'unknown-openbsd'
 194     }
 195
 196     # Consider the direct transformation first and then the special cases
 197     if ostype in ostype_mapper:
 198         ostype = ostype_mapper[ostype]
 199     elif ostype == 'Linux':
 200         os_from_sp = subprocess.check_output(
 201             ['uname', '-o']).strip().decode(default_encoding)
 202         if os_from_sp == 'Android':
 203             ostype = 'linux-android'
 204         else:
 205             ostype = 'unknown-linux-gnu'
 206     elif ostype == 'SunOS':
 207         ostype = 'sun-solaris'
 208         # On Solaris, uname -m will return a machine classification instead
 209         # of a cpu type, so uname -p is recommended instead.  However, the
 210         # output from that option is too generic for our purposes (it will
 211         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 212         # must be used instead.
 213         try:
 214             cputype = subprocess.check_output(
 215                 ['isainfo', '-k']).strip().decode(default_encoding)
 216         except (subprocess.CalledProcessError, OSError):
 217             err = "isainfo not found"
 218             sys.exit(err)
 219     elif ostype.startswith('MINGW'):
 220         # msys' `uname` does not print gcc configuration, but prints msys
 221         # configuration. so we cannot believe `uname -m`:
 222         # msys1 is always i686 and msys2 is always x86_64.
 223         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 224         # MINGW64 on x86_64.
 225         ostype = 'pc-windows-gnu'
 226         cputype = 'i686'
 227         if os.environ.get('MSYSTEM') == 'MINGW64':
 228             cputype = 'x86_64'
 229     elif ostype.startswith('MSYS'):
 230         ostype = 'pc-windows-gnu'
 231     elif ostype.startswith('CYGWIN_NT'):
 232         cputype = 'i686'
 233         if ostype.endswith('WOW64'):
 234             cputype = 'x86_64'
 235         ostype = 'pc-windows-gnu'
 236     else:
 237         err = "unknown OS type: {}".format(ostype)
 238         sys.exit(err)
 239
 240     cputype_mapper = {
 241         'BePC': 'i686',
 242         'aarch64': 'aarch64',
 243         'amd64': 'x86_64',
 244         'arm64': 'aarch64',
 245         'i386': 'i686',
 246         'i486': 'i686',
 247         'i686': 'i686',
 248         'i786': 'i686',
 249         'powerpc': 'powerpc',
 250         'powerpc64': 'powerpc64',
 251         'powerpc64le': 'powerpc64le',
 252         'ppc': 'powerpc',
 253         'ppc64': 'powerpc64',
 254         'ppc64le': 'powerpc64le',
 255         's390x': 's390x',
 256         'x64': 'x86_64',
 257         'x86': 'i686',
 258         'x86-64': 'x86_64',
 259         'x86_64': 'x86_64'
 260     }
 261
 262     # Consider the direct transformation first and then the special cases
 263     if cputype in cputype_mapper:
 264         cputype = cputype_mapper[cputype]
 265     elif cputype in {'xscale', 'arm'}:
 266         cputype = 'arm'
 267         if ostype == 'linux-android':
 268             ostype = 'linux-androideabi'
 269     elif cputype == 'armv6l':
 270         cputype = 'arm'
 271         if ostype == 'linux-android':
 272             ostype = 'linux-androideabi'
 273         else:
 274             ostype += 'eabihf'
 275     elif cputype in {'armv7l', 'armv8l'}:
 276         cputype = 'armv7'
 277         if ostype == 'linux-android':
 278             ostype = 'linux-androideabi'
 279         else:
 280             ostype += 'eabihf'
 281     elif cputype == 'mips':
 282         if sys.byteorder == 'big':
 283             cputype = 'mips'
 284         elif sys.byteorder == 'little':
 285             cputype = 'mipsel'
 286         else:
 287             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 288     elif cputype == 'mips64':
 289         if sys.byteorder == 'big':
 290             cputype = 'mips64'
 291         elif sys.byteorder == 'little':
 292             cputype = 'mips64el'
 293         else:
 294             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 295         # only the n64 ABI is supported, indicate it
 296         ostype += 'abi64'
 297     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 298         pass
 299     else:
 300         err = "unknown cpu type: {}".format(cputype)
 301         sys.exit(err)
 302
 303     return "{}-{}".format(cputype, ostype)
 304
 305
 306 class RustBuild(object):
 307     """Provide all the methods required to build Rust"""
 308     def __init__(self):
 309         self.cargo_channel = ''
 310         self.date = ''
 311         self._download_url = 'https://static.rust-lang.org'
 312         self.rustc_channel = ''
 313         self.build = ''
 314         self.build_dir = os.path.join(os.getcwd(), "build")
 315         self.clean = False
 316         self.config_toml = ''
 317         self.rust_root = ''
 318         self.use_locked_deps = ''
 319         self.use_vendored_sources = ''
 320         self.verbose = False
 321
 322     def download_stage0(self):
 323         """Fetch the build system for Rust, written in Rust
 324
 325         This method will build a cache directory, then it will fetch the
 326         tarball which has the stage0 compiler used to then bootstrap the Rust
 327         compiler itself.
 328
 329         Each downloaded tarball is extracted, after that, the script
 330         will move all the content to the right place.
 331         """
 332         rustc_channel = self.rustc_channel
 333         cargo_channel = self.cargo_channel
 334
 335         if self.rustc().startswith(self.bin_root()) and \
 336                 (not os.path.exists(self.rustc()) or
 337                  self.program_out_of_date(self.rustc_stamp())):
 338             if os.path.exists(self.bin_root()):
 339                 shutil.rmtree(self.bin_root())
 340             filename = "rust-std-{}-{}.tar.gz".format(
 341                 rustc_channel, self.build)
 342             pattern = "rust-std-{}".format(self.build)
 343             self._download_stage0_helper(filename, pattern)
 344
 345             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
 346             self._download_stage0_helper(filename, "rustc")
 347             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 348             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 349             with open(self.rustc_stamp(), 'w') as rust_stamp:
 350                 rust_stamp.write(self.date)
 351
 352             # This is required so that we don't mix incompatible MinGW
 353             # libraries/binaries that are included in rust-std with
 354             # the system MinGW ones.
 355             if "pc-windows-gnu" in self.build:
 356                 filename = "rust-mingw-{}-{}.tar.gz".format(
 357                     rustc_channel, self.build)
 358                 self._download_stage0_helper(filename, "rust-mingw")
 359
 360         if self.cargo().startswith(self.bin_root()) and \
 361                 (not os.path.exists(self.cargo()) or
 362                  self.program_out_of_date(self.cargo_stamp())):
 363             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
 364             self._download_stage0_helper(filename, "cargo")
 365             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 366             with open(self.cargo_stamp(), 'w') as cargo_stamp:
 367                 cargo_stamp.write(self.date)
 368
 369     def _download_stage0_helper(self, filename, pattern):
 370         cache_dst = os.path.join(self.build_dir, "cache")
 371         rustc_cache = os.path.join(cache_dst, self.date)
 372         if not os.path.exists(rustc_cache):
 373             os.makedirs(rustc_cache)
 374
 375         url = "{}/dist/{}".format(self._download_url, self.date)
 376         tarball = os.path.join(rustc_cache, filename)
 377         if not os.path.exists(tarball):
 378             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 379         unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
 380
 381     @staticmethod
 382     def fix_executable(fname):
 383         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 384
 385         This method is only required on NixOS and uses the PatchELF utility to
 386         change the dynamic linker of ELF executables.
 387
 388         Please see https://nixos.org/patchelf.html for more information
 389         """
 390         default_encoding = sys.getdefaultencoding()
 391         try:
 392             ostype = subprocess.check_output(
 393                 ['uname', '-s']).strip().decode(default_encoding)
 394         except subprocess.CalledProcessError:
 395             return
 396         except OSError as reason:
 397             if getattr(reason, 'winerror', None) is not None:
 398                 return
 399             raise reason
 400
 401         if ostype != "Linux":
 402             return
 403
 404         if not os.path.exists("/etc/NIXOS"):
 405             return
 406         if os.path.exists("/lib"):
 407             return
 408
 409         # At this point we're pretty sure the user is running NixOS
 410         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 411         print(nix_os_msg, fname)
 412
 413         try:
 414             interpreter = subprocess.check_output(
 415                 ["patchelf", "--print-interpreter", fname])
 416             interpreter = interpreter.strip().decode(default_encoding)
 417         except subprocess.CalledProcessError as reason:
 418             print("warning: failed to call patchelf:", reason)
 419             return
 420
 421         loader = interpreter.split("/")[-1]
 422
 423         try:
 424             ldd_output = subprocess.check_output(
 425                 ['ldd', '/run/current-system/sw/bin/sh'])
 426             ldd_output = ldd_output.strip().decode(default_encoding)
 427         except subprocess.CalledProcessError as reason:
 428             print("warning: unable to call ldd:", reason)
 429             return
 430
 431         for line in ldd_output.splitlines():
 432             libname = line.split()[0]
 433             if libname.endswith(loader):
 434                 loader_path = libname[:len(libname) - len(loader)]
 435                 break
 436         else:
 437             print("warning: unable to find the path to the dynamic linker")
 438             return
 439
 440         correct_interpreter = loader_path + loader
 441
 442         try:
 443             subprocess.check_output(
 444                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 445         except subprocess.CalledProcessError as reason:
 446             print("warning: failed to call patchelf:", reason)
 447             return
 448
 449     def rustc_stamp(self):
 450         """Return the path for .rustc-stamp
 451
 452         >>> rb = RustBuild()
 453         >>> rb.build_dir = "build"
 454         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 455         True
 456         """
 457         return os.path.join(self.bin_root(), '.rustc-stamp')
 458
 459     def cargo_stamp(self):
 460         """Return the path for .cargo-stamp
 461
 462         >>> rb = RustBuild()
 463         >>> rb.build_dir = "build"
 464         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 465         True
 466         """
 467         return os.path.join(self.bin_root(), '.cargo-stamp')
 468
 469     def program_out_of_date(self, stamp_path):
 470         """Check if the given program stamp is out of date"""
 471         if not os.path.exists(stamp_path) or self.clean:
 472             return True
 473         with open(stamp_path, 'r') as stamp:
 474             return self.date != stamp.read()
 475
 476     def bin_root(self):
 477         """Return the binary root directory
 478
 479         >>> rb = RustBuild()
 480         >>> rb.build_dir = "build"
 481         >>> rb.bin_root() == os.path.join("build", "stage0")
 482         True
 483
 484         When the 'build' property is given should be a nested directory:
 485
 486         >>> rb.build = "devel"
 487         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 488         True
 489         """
 490         return os.path.join(self.build_dir, self.build, "stage0")
 491
 492     def get_toml(self, key, section=None):
 493         """Returns the value of the given key in config.toml, otherwise returns None
 494
 495         >>> rb = RustBuild()
 496         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 497         >>> rb.get_toml("key2")
 498         'value2'
 499
 500         If the key does not exists, the result is None:
 501
 502         >>> rb.get_toml("key3") is None
 503         True
 504
 505         Optionally also matches the section the key appears in
 506
 507         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 508         >>> rb.get_toml('key', 'a')
 509         'value1'
 510         >>> rb.get_toml('key', 'b')
 511         'value2'
 512         >>> rb.get_toml('key', 'c') is None
 513         True
 514         """
 515
 516         cur_section = None
 517         for line in self.config_toml.splitlines():
 518             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 519             if section_match is not None:
 520                 cur_section = section_match.group(1)
 521
 522             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 523             if match is not None:
 524                 value = match.group(1)
 525                 if section is None or section == cur_section:
 526                     return self.get_string(value) or value.strip()
 527         return None
 528
 529     def cargo(self):
 530         """Return config path for cargo"""
 531         return self.program_config('cargo')
 532
 533     def rustc(self):
 534         """Return config path for rustc"""
 535         return self.program_config('rustc')
 536
 537     def program_config(self, program):
 538         """Return config path for the given program
 539
 540         >>> rb = RustBuild()
 541         >>> rb.config_toml = 'rustc = "rustc"\\n'
 542         >>> rb.program_config('rustc')
 543         'rustc'
 544         >>> rb.config_toml = ''
 545         >>> cargo_path = rb.program_config('cargo')
 546         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 547         ... "bin", "cargo")
 548         True
 549         """
 550         config = self.get_toml(program)
 551         if config:
 552             return os.path.expanduser(config)
 553         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 554             program, self.exe_suffix()))
 555
 556     @staticmethod
 557     def get_string(line):
 558         """Return the value between double quotes
 559
 560         >>> RustBuild.get_string('    "devel"   ')
 561         'devel'
 562         """
 563         start = line.find('"')
 564         if start != -1:
 565             end = start + 1 + line[start + 1:].find('"')
 566             return line[start + 1:end]
 567         start = line.find('\'')
 568         if start != -1:
 569             end = start + 1 + line[start + 1:].find('\'')
 570             return line[start + 1:end]
 571         return None
 572
 573     @staticmethod
 574     def exe_suffix():
 575         """Return a suffix for executables"""
 576         if sys.platform == 'win32':
 577             return '.exe'
 578         return ''
 579
 580     def bootstrap_binary(self):
 581         """Return the path of the boostrap binary
 582
 583         >>> rb = RustBuild()
 584         >>> rb.build_dir = "build"
 585         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 586         ... "debug", "bootstrap")
 587         True
 588         """
 589         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 590
 591     def build_bootstrap(self):
 592         """Build bootstrap"""
 593         build_dir = os.path.join(self.build_dir, "bootstrap")
 594         if self.clean and os.path.exists(build_dir):
 595             shutil.rmtree(build_dir)
 596         env = os.environ.copy()
 597         env["RUSTC_BOOTSTRAP"] = '1'
 598         env["CARGO_TARGET_DIR"] = build_dir
 599         env["RUSTC"] = self.rustc()
 600         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 601             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 602             if "LD_LIBRARY_PATH" in env else ""
 603         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 604             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 605             if "DYLD_LIBRARY_PATH" in env else ""
 606         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 607             (os.pathsep + env["LIBRARY_PATH"]) \
 608             if "LIBRARY_PATH" in env else ""
 609         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 610
 611         build_section = "target.{}".format(self.build_triple())
 612         target_features = []
 613         if self.get_toml("crt-static", build_section) == "true":
 614             target_features += ["+crt-static"]
 615         elif self.get_toml("crt-static", build_section) == "false":
 616             target_features += ["-crt-static"]
 617         if target_features:
 618             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 619
 620         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 621             os.pathsep + env["PATH"]
 622         if not os.path.isfile(self.cargo()):
 623             raise Exception("no cargo executable found at `{}`".format(
 624                 self.cargo()))
 625         args = [self.cargo(), "build", "--manifest-path",
 626                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 627         for _ in range(1, self.verbose):
 628             args.append("--verbose")
 629         if self.use_locked_deps:
 630             args.append("--locked")
 631         if self.use_vendored_sources:
 632             args.append("--frozen")
 633         run(args, env=env, verbose=self.verbose)
 634
 635     def build_triple(self):
 636         """Build triple as in LLVM"""
 637         config = self.get_toml('build')
 638         if config:
 639             return config
 640         return default_build_triple()
 641
 642     def check_submodule(self, module, slow_submodules):
 643         if not slow_submodules:
 644             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 645                                            cwd=os.path.join(self.rust_root, module),
 646                                            stdout=subprocess.PIPE)
 647             return checked_out
 648         else:
 649             return None
 650
 651     def update_submodule(self, module, checked_out, recorded_submodules):
 652         module_path = os.path.join(self.rust_root, module)
 653
 654         if checked_out != None:
 655             default_encoding = sys.getdefaultencoding()
 656             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 657             if recorded_submodules[module] == checked_out:
 658                 return
 659
 660         print("Updating submodule", module)
 661
 662         run(["git", "submodule", "-q", "sync", module],
 663             cwd=self.rust_root, verbose=self.verbose)
 664         run(["git", "submodule", "update",
 665             "--init", "--recursive", module],
 666             cwd=self.rust_root, verbose=self.verbose)
 667         run(["git", "reset", "-q", "--hard"],
 668             cwd=module_path, verbose=self.verbose)
 669         run(["git", "clean", "-qdfx"],
 670             cwd=module_path, verbose=self.verbose)
 671
 672     def update_submodules(self):
 673         """Update submodules"""
 674         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 675                 self.get_toml('submodules') == "false":
 676             return
 677         slow_submodules = self.get_toml('fast-submodules') == "false"
 678         start_time = time()
 679         if slow_submodules:
 680             print('Unconditionally updating all submodules')
 681         else:
 682             print('Updating only changed submodules')
 683         default_encoding = sys.getdefaultencoding()
 684         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 685             ["git", "config", "--file",
 686              os.path.join(self.rust_root, ".gitmodules"),
 687              "--get-regexp", "path"]
 688         ).decode(default_encoding).splitlines()]
 689         filtered_submodules = []
 690         submodules_names = []
 691         for module in submodules:
 692             if module.endswith("llvm"):
 693                 if self.get_toml('llvm-config'):
 694                     continue
 695             if module.endswith("llvm-emscripten"):
 696                 backends = self.get_toml('codegen-backends')
 697                 if backends is None or not 'emscripten' in backends:
 698                     continue
 699             if module.endswith("jemalloc"):
 700                 if self.get_toml('use-jemalloc') == 'false':
 701                     continue
 702                 if self.get_toml('jemalloc'):
 703                     continue
 704             if module.endswith("lld"):
 705                 config = self.get_toml('lld')
 706                 if config is None or config == 'false':
 707                     continue
 708             check = self.check_submodule(module, slow_submodules)
 709             filtered_submodules.append((module, check))
 710             submodules_names.append(module)
 711         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 712                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 713         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 714         recorded_submodules = {}
 715         for data in recorded:
 716             data = data.split()
 717             recorded_submodules[data[3]] = data[2]
 718         for module in filtered_submodules:
 719             self.update_submodule(module[0], module[1], recorded_submodules)
 720         print("Submodules updated in %.2f seconds" % (time() - start_time))
 721
 722     def set_dev_environment(self):
 723         """Set download URL for development environment"""
 724         self._download_url = 'https://dev-static.rust-lang.org'
 725
 726
 727 def bootstrap(help_triggered):
 728     """Configure, fetch, build and run the initial bootstrap"""
 729
 730     # If the user is asking for help, let them know that the whole download-and-build
 731     # process has to happen before anything is printed out.
 732     if help_triggered:
 733         print("info: Downloading and building bootstrap before processing --help")
 734         print("      command. See src/bootstrap/README.md for help with common")
 735         print("      commands.")
 736
 737     parser = argparse.ArgumentParser(description='Build rust')
 738     parser.add_argument('--config')
 739     parser.add_argument('--build')
 740     parser.add_argument('--src')
 741     parser.add_argument('--clean', action='store_true')
 742     parser.add_argument('-v', '--verbose', action='count', default=0)
 743
 744     args = [a for a in sys.argv if a != '-h' and a != '--help']
 745     args, _ = parser.parse_known_args(args)
 746
 747     # Configure initial bootstrap
 748     build = RustBuild()
 749     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 750     build.verbose = args.verbose
 751     build.clean = args.clean
 752
 753     try:
 754         with open(args.config or 'config.toml') as config:
 755             build.config_toml = config.read()
 756     except (OSError, IOError):
 757         pass
 758
 759     match = re.search(r'\nverbose = (\d+)', build.config_toml)
 760     if match is not None:
 761         build.verbose = max(build.verbose, int(match.group(1)))
 762
 763     build.use_vendored_sources = '\nvendor = true' in build.config_toml
 764
 765     build.use_locked_deps = '\nlocked-deps = true' in build.config_toml
 766
 767     if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
 768         if os.environ.get('USER') != os.environ['SUDO_USER']:
 769             build.use_vendored_sources = True
 770             print('info: looks like you are running this command under `sudo`')
 771             print('      and so in order to preserve your $HOME this will now')
 772             print('      use vendored sources by default. Note that if this')
 773             print('      does not work you should run a normal build first')
 774             print('      before running a command like `sudo ./x.py install`')
 775
 776     if build.use_vendored_sources:
 777         if not os.path.exists('.cargo'):
 778             os.makedirs('.cargo')
 779         with open('.cargo/config', 'w') as cargo_config:
 780             cargo_config.write("""
 781                 [source.crates-io]
 782                 replace-with = 'vendored-sources'
 783                 registry = 'https://example.com'
 784
 785                 [source.vendored-sources]
 786                 directory = '{}/src/vendor'
 787             """.format(build.rust_root))
 788     else:
 789         if os.path.exists('.cargo'):
 790             shutil.rmtree('.cargo')
 791
 792     data = stage0_data(build.rust_root)
 793     build.date = data['date']
 794     build.rustc_channel = data['rustc']
 795     build.cargo_channel = data['cargo']
 796
 797     if 'dev' in data:
 798         build.set_dev_environment()
 799
 800     build.update_submodules()
 801
 802     # Fetch/build the bootstrap
 803     build.build = args.build or build.build_triple()
 804     build.download_stage0()
 805     sys.stdout.flush()
 806     build.build_bootstrap()
 807     sys.stdout.flush()
 808
 809     # Run the bootstrap
 810     args = [build.bootstrap_binary()]
 811     args.extend(sys.argv[1:])
 812     env = os.environ.copy()
 813     env["BUILD"] = build.build
 814     env["SRC"] = build.rust_root
 815     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 816     env["BOOTSTRAP_PYTHON"] = sys.executable
 817     env["BUILD_DIR"] = build.build_dir
 818     env["RUSTC_BOOTSTRAP"] = '1'
 819     env["CARGO"] = build.cargo()
 820     env["RUSTC"] = build.rustc()
 821     run(args, env=env, verbose=build.verbose)
 822
 823
 824 def main():
 825     """Entry point for the bootstrap process"""
 826     start_time = time()
 827     help_triggered = (
 828         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 829     try:
 830         bootstrap(help_triggered)
 831         if not help_triggered:
 832             print("Build completed successfully in {}".format(
 833                 format_build_time(time() - start_time)))
 834     except (SystemExit, KeyboardInterrupt) as error:
 835         if hasattr(error, 'code') and isinstance(error.code, int):
 836             exit_code = error.code
 837         else:
 838             exit_code = 1
 839             print(error)
 840         if not help_triggered:
 841             print("Build completed unsuccessfully in {}".format(
 842                 format_build_time(time() - start_time)))
 843         sys.exit(exit_code)
 844
 845
 846 if __name__ == '__main__':
 847     main()