src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = ''
 324         self.rustc_channel = ''
 325         self.rustfmt_channel = ''
 326         self.build = ''
 327         self.build_dir = os.path.join(os.getcwd(), "build")
 328         self.clean = False
 329         self.config_toml = ''
 330         self.rust_root = ''
 331         self.use_locked_deps = ''
 332         self.use_vendored_sources = ''
 333         self.verbose = False
 334
 335     def download_stage0(self):
 336         """Fetch the build system for Rust, written in Rust
 337
 338         This method will build a cache directory, then it will fetch the
 339         tarball which has the stage0 compiler used to then bootstrap the Rust
 340         compiler itself.
 341
 342         Each downloaded tarball is extracted, after that, the script
 343         will move all the content to the right place.
 344         """
 345         rustc_channel = self.rustc_channel
 346         cargo_channel = self.cargo_channel
 347         rustfmt_channel = self.rustfmt_channel
 348
 349         def support_xz():
 350             try:
 351                 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
 352                     temp_path = temp_file.name
 353                 with tarfile.open(temp_path, "w:xz"):
 354                     pass
 355                 return True
 356             except tarfile.CompressionError:
 357                 return False
 358
 359         if self.rustc().startswith(self.bin_root()) and \
 360                 (not os.path.exists(self.rustc()) or
 361                  self.program_out_of_date(self.rustc_stamp())):
 362             if os.path.exists(self.bin_root()):
 363                 shutil.rmtree(self.bin_root())
 364             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 365             filename = "rust-std-{}-{}{}".format(
 366                 rustc_channel, self.build, tarball_suffix)
 367             pattern = "rust-std-{}".format(self.build)
 368             self._download_stage0_helper(filename, pattern, tarball_suffix)
 369
 370             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 371                                               tarball_suffix)
 372             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 373             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 374             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 375             with output(self.rustc_stamp()) as rust_stamp:
 376                 rust_stamp.write(self.date)
 377
 378             # This is required so that we don't mix incompatible MinGW
 379             # libraries/binaries that are included in rust-std with
 380             # the system MinGW ones.
 381             if "pc-windows-gnu" in self.build:
 382                 filename = "rust-mingw-{}-{}{}".format(
 383                     rustc_channel, self.build, tarball_suffix)
 384                 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
 385
 386         if self.cargo().startswith(self.bin_root()) and \
 387                 (not os.path.exists(self.cargo()) or
 388                  self.program_out_of_date(self.cargo_stamp())):
 389             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 390             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 391                                               tarball_suffix)
 392             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 393             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 394             with output(self.cargo_stamp()) as cargo_stamp:
 395                 cargo_stamp.write(self.date)
 396
 397         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 398             not os.path.exists(self.rustfmt())
 399             or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
 400         ):
 401             if rustfmt_channel:
 402                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 403                 [channel, date] = rustfmt_channel.split('-', 1)
 404                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 405                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 406                 self.fix_executable("{}/bin/rustfmt".format(self.bin_root()))
 407                 self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root()))
 408                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 409                     rustfmt_stamp.write(self.date + self.rustfmt_channel)
 410
 411     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 412         if date is None:
 413             date = self.date
 414         cache_dst = os.path.join(self.build_dir, "cache")
 415         rustc_cache = os.path.join(cache_dst, date)
 416         if not os.path.exists(rustc_cache):
 417             os.makedirs(rustc_cache)
 418
 419         url = "{}/dist/{}".format(self._download_url, date)
 420         tarball = os.path.join(rustc_cache, filename)
 421         if not os.path.exists(tarball):
 422             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 423         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 424
 425     @staticmethod
 426     def fix_executable(fname):
 427         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 428
 429         This method is only required on NixOS and uses the PatchELF utility to
 430         change the dynamic linker of ELF executables.
 431
 432         Please see https://nixos.org/patchelf.html for more information
 433         """
 434         default_encoding = sys.getdefaultencoding()
 435         try:
 436             ostype = subprocess.check_output(
 437                 ['uname', '-s']).strip().decode(default_encoding)
 438         except subprocess.CalledProcessError:
 439             return
 440         except OSError as reason:
 441             if getattr(reason, 'winerror', None) is not None:
 442                 return
 443             raise reason
 444
 445         if ostype != "Linux":
 446             return
 447
 448         if not os.path.exists("/etc/NIXOS"):
 449             return
 450         if os.path.exists("/lib"):
 451             return
 452
 453         # At this point we're pretty sure the user is running NixOS
 454         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 455         print(nix_os_msg, fname)
 456
 457         try:
 458             interpreter = subprocess.check_output(
 459                 ["patchelf", "--print-interpreter", fname])
 460             interpreter = interpreter.strip().decode(default_encoding)
 461         except subprocess.CalledProcessError as reason:
 462             print("warning: failed to call patchelf:", reason)
 463             return
 464
 465         loader = interpreter.split("/")[-1]
 466
 467         try:
 468             ldd_output = subprocess.check_output(
 469                 ['ldd', '/run/current-system/sw/bin/sh'])
 470             ldd_output = ldd_output.strip().decode(default_encoding)
 471         except subprocess.CalledProcessError as reason:
 472             print("warning: unable to call ldd:", reason)
 473             return
 474
 475         for line in ldd_output.splitlines():
 476             libname = line.split()[0]
 477             if libname.endswith(loader):
 478                 loader_path = libname[:len(libname) - len(loader)]
 479                 break
 480         else:
 481             print("warning: unable to find the path to the dynamic linker")
 482             return
 483
 484         correct_interpreter = loader_path + loader
 485
 486         try:
 487             subprocess.check_output(
 488                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 489         except subprocess.CalledProcessError as reason:
 490             print("warning: failed to call patchelf:", reason)
 491             return
 492
 493     def rustc_stamp(self):
 494         """Return the path for .rustc-stamp
 495
 496         >>> rb = RustBuild()
 497         >>> rb.build_dir = "build"
 498         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 499         True
 500         """
 501         return os.path.join(self.bin_root(), '.rustc-stamp')
 502
 503     def cargo_stamp(self):
 504         """Return the path for .cargo-stamp
 505
 506         >>> rb = RustBuild()
 507         >>> rb.build_dir = "build"
 508         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 509         True
 510         """
 511         return os.path.join(self.bin_root(), '.cargo-stamp')
 512
 513     def rustfmt_stamp(self):
 514         """Return the path for .rustfmt-stamp
 515
 516         >>> rb = RustBuild()
 517         >>> rb.build_dir = "build"
 518         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 519         True
 520         """
 521         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 522
 523     def program_out_of_date(self, stamp_path, extra=""):
 524         """Check if the given program stamp is out of date"""
 525         if not os.path.exists(stamp_path) or self.clean:
 526             return True
 527         with open(stamp_path, 'r') as stamp:
 528             return (self.date + extra) != stamp.read()
 529
 530     def bin_root(self):
 531         """Return the binary root directory
 532
 533         >>> rb = RustBuild()
 534         >>> rb.build_dir = "build"
 535         >>> rb.bin_root() == os.path.join("build", "stage0")
 536         True
 537
 538         When the 'build' property is given should be a nested directory:
 539
 540         >>> rb.build = "devel"
 541         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 542         True
 543         """
 544         return os.path.join(self.build_dir, self.build, "stage0")
 545
 546     def get_toml(self, key, section=None):
 547         """Returns the value of the given key in config.toml, otherwise returns None
 548
 549         >>> rb = RustBuild()
 550         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 551         >>> rb.get_toml("key2")
 552         'value2'
 553
 554         If the key does not exists, the result is None:
 555
 556         >>> rb.get_toml("key3") is None
 557         True
 558
 559         Optionally also matches the section the key appears in
 560
 561         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 562         >>> rb.get_toml('key', 'a')
 563         'value1'
 564         >>> rb.get_toml('key', 'b')
 565         'value2'
 566         >>> rb.get_toml('key', 'c') is None
 567         True
 568
 569         >>> rb.config_toml = 'key1 = true'
 570         >>> rb.get_toml("key1")
 571         'true'
 572         """
 573
 574         cur_section = None
 575         for line in self.config_toml.splitlines():
 576             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 577             if section_match is not None:
 578                 cur_section = section_match.group(1)
 579
 580             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 581             if match is not None:
 582                 value = match.group(1)
 583                 if section is None or section == cur_section:
 584                     return self.get_string(value) or value.strip()
 585         return None
 586
 587     def cargo(self):
 588         """Return config path for cargo"""
 589         return self.program_config('cargo')
 590
 591     def rustc(self):
 592         """Return config path for rustc"""
 593         return self.program_config('rustc')
 594
 595     def rustfmt(self):
 596         """Return config path for rustfmt"""
 597         if not self.rustfmt_channel:
 598             return None
 599         return self.program_config('rustfmt')
 600
 601     def program_config(self, program):
 602         """Return config path for the given program
 603
 604         >>> rb = RustBuild()
 605         >>> rb.config_toml = 'rustc = "rustc"\\n'
 606         >>> rb.program_config('rustc')
 607         'rustc'
 608         >>> rb.config_toml = ''
 609         >>> cargo_path = rb.program_config('cargo')
 610         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 611         ... "bin", "cargo")
 612         True
 613         """
 614         config = self.get_toml(program)
 615         if config:
 616             return os.path.expanduser(config)
 617         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 618             program, self.exe_suffix()))
 619
 620     @staticmethod
 621     def get_string(line):
 622         """Return the value between double quotes
 623
 624         >>> RustBuild.get_string('    "devel"   ')
 625         'devel'
 626         >>> RustBuild.get_string("    'devel'   ")
 627         'devel'
 628         >>> RustBuild.get_string('devel') is None
 629         True
 630         >>> RustBuild.get_string('    "devel   ')
 631         ''
 632         """
 633         start = line.find('"')
 634         if start != -1:
 635             end = start + 1 + line[start + 1:].find('"')
 636             return line[start + 1:end]
 637         start = line.find('\'')
 638         if start != -1:
 639             end = start + 1 + line[start + 1:].find('\'')
 640             return line[start + 1:end]
 641         return None
 642
 643     @staticmethod
 644     def exe_suffix():
 645         """Return a suffix for executables"""
 646         if sys.platform == 'win32':
 647             return '.exe'
 648         return ''
 649
 650     def bootstrap_binary(self):
 651         """Return the path of the bootstrap binary
 652
 653         >>> rb = RustBuild()
 654         >>> rb.build_dir = "build"
 655         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 656         ... "debug", "bootstrap")
 657         True
 658         """
 659         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 660
 661     def build_bootstrap(self):
 662         """Build bootstrap"""
 663         build_dir = os.path.join(self.build_dir, "bootstrap")
 664         if self.clean and os.path.exists(build_dir):
 665             shutil.rmtree(build_dir)
 666         env = os.environ.copy()
 667         env["RUSTC_BOOTSTRAP"] = '1'
 668         env["CARGO_TARGET_DIR"] = build_dir
 669         env["RUSTC"] = self.rustc()
 670         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 671             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 672             if "LD_LIBRARY_PATH" in env else ""
 673         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 674             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 675             if "DYLD_LIBRARY_PATH" in env else ""
 676         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 677             (os.pathsep + env["LIBRARY_PATH"]) \
 678             if "LIBRARY_PATH" in env else ""
 679         # preserve existing RUSTFLAGS
 680         env.setdefault("RUSTFLAGS", "")
 681         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 682
 683         build_section = "target.{}".format(self.build_triple())
 684         target_features = []
 685         if self.get_toml("crt-static", build_section) == "true":
 686             target_features += ["+crt-static"]
 687         elif self.get_toml("crt-static", build_section) == "false":
 688             target_features += ["-crt-static"]
 689         if target_features:
 690             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 691         target_linker = self.get_toml("linker", build_section)
 692         if target_linker is not None:
 693             env["RUSTFLAGS"] += " -C linker=" + target_linker
 694         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 695         if self.get_toml("deny-warnings", "rust") != "false":
 696             env["RUSTFLAGS"] += " -Dwarnings"
 697
 698         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 699             os.pathsep + env["PATH"]
 700         if not os.path.isfile(self.cargo()):
 701             raise Exception("no cargo executable found at `{}`".format(
 702                 self.cargo()))
 703         args = [self.cargo(), "build", "--manifest-path",
 704                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 705         for _ in range(1, self.verbose):
 706             args.append("--verbose")
 707         if self.use_locked_deps:
 708             args.append("--locked")
 709         if self.use_vendored_sources:
 710             args.append("--frozen")
 711         run(args, env=env, verbose=self.verbose)
 712
 713     def build_triple(self):
 714         """Build triple as in LLVM"""
 715         config = self.get_toml('build')
 716         if config:
 717             return config
 718         return default_build_triple()
 719
 720     def check_submodule(self, module, slow_submodules):
 721         if not slow_submodules:
 722             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 723                                            cwd=os.path.join(self.rust_root, module),
 724                                            stdout=subprocess.PIPE)
 725             return checked_out
 726         else:
 727             return None
 728
 729     def update_submodule(self, module, checked_out, recorded_submodules):
 730         module_path = os.path.join(self.rust_root, module)
 731
 732         if checked_out is not None:
 733             default_encoding = sys.getdefaultencoding()
 734             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 735             if recorded_submodules[module] == checked_out:
 736                 return
 737
 738         print("Updating submodule", module)
 739
 740         run(["git", "submodule", "-q", "sync", module],
 741             cwd=self.rust_root, verbose=self.verbose)
 742         try:
 743             run(["git", "submodule", "update",
 744                  "--init", "--recursive", "--progress", module],
 745                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 746         except RuntimeError:
 747             # Some versions of git don't support --progress.
 748             run(["git", "submodule", "update",
 749                  "--init", "--recursive", module],
 750                 cwd=self.rust_root, verbose=self.verbose)
 751         run(["git", "reset", "-q", "--hard"],
 752             cwd=module_path, verbose=self.verbose)
 753         run(["git", "clean", "-qdfx"],
 754             cwd=module_path, verbose=self.verbose)
 755
 756     def update_submodules(self):
 757         """Update submodules"""
 758         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 759                 self.get_toml('submodules') == "false":
 760             return
 761
 762         # check the existence of 'git' command
 763         try:
 764             subprocess.check_output(['git', '--version'])
 765         except (subprocess.CalledProcessError, OSError):
 766             print("error: `git` is not found, please make sure it's installed and in the path.")
 767             sys.exit(1)
 768
 769         slow_submodules = self.get_toml('fast-submodules') == "false"
 770         start_time = time()
 771         if slow_submodules:
 772             print('Unconditionally updating all submodules')
 773         else:
 774             print('Updating only changed submodules')
 775         default_encoding = sys.getdefaultencoding()
 776         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 777             ["git", "config", "--file",
 778              os.path.join(self.rust_root, ".gitmodules"),
 779              "--get-regexp", "path"]
 780         ).decode(default_encoding).splitlines()]
 781         filtered_submodules = []
 782         submodules_names = []
 783         for module in submodules:
 784             if module.endswith("llvm-project"):
 785                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 786                     continue
 787             check = self.check_submodule(module, slow_submodules)
 788             filtered_submodules.append((module, check))
 789             submodules_names.append(module)
 790         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 791                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 792         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 793         recorded_submodules = {}
 794         for data in recorded:
 795             data = data.split()
 796             recorded_submodules[data[3]] = data[2]
 797         for module in filtered_submodules:
 798             self.update_submodule(module[0], module[1], recorded_submodules)
 799         print("Submodules updated in %.2f seconds" % (time() - start_time))
 800
 801     def set_normal_environment(self):
 802         """Set download URL for normal environment"""
 803         if 'RUSTUP_DIST_SERVER' in os.environ:
 804             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 805         else:
 806             self._download_url = 'https://static.rust-lang.org'
 807
 808     def set_dev_environment(self):
 809         """Set download URL for development environment"""
 810         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 811             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 812         else:
 813             self._download_url = 'https://dev-static.rust-lang.org'
 814
 815     def check_vendored_status(self):
 816         """Check that vendoring is configured properly"""
 817         vendor_dir = os.path.join(self.rust_root, 'vendor')
 818         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 819             if os.environ.get('USER') != os.environ['SUDO_USER']:
 820                 self.use_vendored_sources = True
 821                 print('info: looks like you are running this command under `sudo`')
 822                 print('      and so in order to preserve your $HOME this will now')
 823                 print('      use vendored sources by default.')
 824                 if not os.path.exists(vendor_dir):
 825                     print('error: vendoring required, but vendor directory does not exist.')
 826                     print('       Run `cargo vendor` without sudo to initialize the '
 827                           'vendor directory.')
 828                     raise Exception("{} not found".format(vendor_dir))
 829
 830         if self.use_vendored_sources:
 831             if not os.path.exists('.cargo'):
 832                 os.makedirs('.cargo')
 833             with output('.cargo/config') as cargo_config:
 834                 cargo_config.write(
 835                     "[source.crates-io]\n"
 836                     "replace-with = 'vendored-sources'\n"
 837                     "registry = 'https://example.com'\n"
 838                     "\n"
 839                     "[source.vendored-sources]\n"
 840                     "directory = '{}/vendor'\n"
 841                     .format(self.rust_root))
 842         else:
 843             if os.path.exists('.cargo'):
 844                 shutil.rmtree('.cargo')
 845
 846     def ensure_vendored(self):
 847         """Ensure that the vendored sources are available if needed"""
 848         vendor_dir = os.path.join(self.rust_root, 'vendor')
 849         # Note that this does not handle updating the vendored dependencies if
 850         # the rust git repository is updated. Normal development usually does
 851         # not use vendoring, so hopefully this isn't too much of a problem.
 852         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 853             run([self.cargo(), "vendor"],
 854                 verbose=self.verbose, cwd=self.rust_root)
 855
 856
 857 def bootstrap(help_triggered):
 858     """Configure, fetch, build and run the initial bootstrap"""
 859
 860     # If the user is asking for help, let them know that the whole download-and-build
 861     # process has to happen before anything is printed out.
 862     if help_triggered:
 863         print("info: Downloading and building bootstrap before processing --help")
 864         print("      command. See src/bootstrap/README.md for help with common")
 865         print("      commands.")
 866
 867     parser = argparse.ArgumentParser(description='Build rust')
 868     parser.add_argument('--config')
 869     parser.add_argument('--build')
 870     parser.add_argument('--src')
 871     parser.add_argument('--clean', action='store_true')
 872     parser.add_argument('-v', '--verbose', action='count', default=0)
 873
 874     args = [a for a in sys.argv if a != '-h' and a != '--help']
 875     args, _ = parser.parse_known_args(args)
 876
 877     # Configure initial bootstrap
 878     build = RustBuild()
 879     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 880     build.verbose = args.verbose
 881     build.clean = args.clean
 882
 883     try:
 884         with open(args.config or 'config.toml') as config:
 885             build.config_toml = config.read()
 886     except (OSError, IOError):
 887         pass
 888
 889     config_verbose = build.get_toml('verbose', 'build')
 890     if config_verbose is not None:
 891         build.verbose = max(build.verbose, int(config_verbose))
 892
 893     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 894
 895     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 896
 897     build.check_vendored_status()
 898
 899     data = stage0_data(build.rust_root)
 900     build.date = data['date']
 901     build.rustc_channel = data['rustc']
 902     build.cargo_channel = data['cargo']
 903
 904     if "rustfmt" in data:
 905         build.rustfmt_channel = data['rustfmt']
 906
 907     if 'dev' in data:
 908         build.set_dev_environment()
 909     else:
 910         build.set_normal_environment()
 911
 912     build.update_submodules()
 913
 914     # Fetch/build the bootstrap
 915     build.build = args.build or build.build_triple()
 916     build.download_stage0()
 917     sys.stdout.flush()
 918     build.ensure_vendored()
 919     build.build_bootstrap()
 920     sys.stdout.flush()
 921
 922     # Run the bootstrap
 923     args = [build.bootstrap_binary()]
 924     args.extend(sys.argv[1:])
 925     env = os.environ.copy()
 926     env["BUILD"] = build.build
 927     env["SRC"] = build.rust_root
 928     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 929     env["BOOTSTRAP_PYTHON"] = sys.executable
 930     env["BUILD_DIR"] = build.build_dir
 931     env["RUSTC_BOOTSTRAP"] = '1'
 932     env["CARGO"] = build.cargo()
 933     env["RUSTC"] = build.rustc()
 934     if build.rustfmt():
 935         env["RUSTFMT"] = build.rustfmt()
 936     run(args, env=env, verbose=build.verbose)
 937
 938
 939 def main():
 940     """Entry point for the bootstrap process"""
 941     start_time = time()
 942
 943     # x.py help <cmd> ...
 944     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 945         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 946
 947     help_triggered = (
 948         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 949     try:
 950         bootstrap(help_triggered)
 951         if not help_triggered:
 952             print("Build completed successfully in {}".format(
 953                 format_build_time(time() - start_time)))
 954     except (SystemExit, KeyboardInterrupt) as error:
 955         if hasattr(error, 'code') and isinstance(error.code, int):
 956             exit_code = error.code
 957         else:
 958             exit_code = 1
 959             print(error)
 960         if not help_triggered:
 961             print("Build completed unsuccessfully in {}".format(
 962                 format_build_time(time() - start_time)))
 963         sys.exit(exit_code)
 964
 965
 966 if __name__ == '__main__':
 967     main()