src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import argparse
  12 import contextlib
  13 import datetime
  14 import hashlib
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tarfile
  20 import tempfile
  21
  22 from time import time
  23
  24
  25 def get(url, path, verbose=False):
  26     sha_url = url + ".sha256"
  27     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  28         temp_path = temp_file.name
  29     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  30         sha_path = sha_file.name
  31
  32     try:
  33         download(sha_path, sha_url, False, verbose)
  34         if os.path.exists(path):
  35             if verify(path, sha_path, False):
  36                 if verbose:
  37                     print("using already-download file " + path)
  38                 return
  39             else:
  40                 if verbose:
  41                     print("ignoring already-download file " + path + " due to failed verification")
  42                 os.unlink(path)
  43         download(temp_path, url, True, verbose)
  44         if not verify(temp_path, sha_path, verbose):
  45             raise RuntimeError("failed verification")
  46         if verbose:
  47             print("moving {} to {}".format(temp_path, path))
  48         shutil.move(temp_path, path)
  49     finally:
  50         delete_if_present(sha_path, verbose)
  51         delete_if_present(temp_path, verbose)
  52
  53
  54 def delete_if_present(path, verbose):
  55     if os.path.isfile(path):
  56         if verbose:
  57             print("removing " + path)
  58         os.unlink(path)
  59
  60
  61 def download(path, url, probably_big, verbose):
  62     if probably_big or verbose:
  63         print("downloading {}".format(url))
  64     # see http://serverfault.com/questions/301128/how-to-download
  65     if sys.platform == 'win32':
  66         run(["PowerShell.exe", "/nologo", "-Command",
  67              "(New-Object System.Net.WebClient)"
  68              ".DownloadFile('{}', '{}')".format(url, path)],
  69             verbose=verbose)
  70     else:
  71         if probably_big or verbose:
  72             option = "-#"
  73         else:
  74             option = "-s"
  75         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url], verbose=verbose)
  76
  77
  78 def verify(path, sha_path, verbose):
  79     if verbose:
  80         print("verifying " + path)
  81     with open(path, "rb") as f:
  82         found = hashlib.sha256(f.read()).hexdigest()
  83     with open(sha_path, "r") as f:
  84         expected = f.readline().split()[0]
  85     verified = found == expected
  86     if not verified:
  87         print("invalid checksum:\n"
  88                "    found:    {}\n"
  89                "    expected: {}".format(found, expected))
  90     return verified
  91
  92
  93 def unpack(tarball, dst, verbose=False, match=None):
  94     print("extracting " + tarball)
  95     fname = os.path.basename(tarball).replace(".tar.gz", "")
  96     with contextlib.closing(tarfile.open(tarball)) as tar:
  97         for p in tar.getnames():
  98             if "/" not in p:
  99                 continue
 100             name = p.replace(fname + "/", "", 1)
 101             if match is not None and not name.startswith(match):
 102                 continue
 103             name = name[len(match) + 1:]
 104
 105             fp = os.path.join(dst, name)
 106             if verbose:
 107                 print("  extracting " + p)
 108             tar.extract(p, dst)
 109             tp = os.path.join(dst, p)
 110             if os.path.isdir(tp) and os.path.exists(fp):
 111                 continue
 112             shutil.move(tp, fp)
 113     shutil.rmtree(os.path.join(dst, fname))
 114
 115 def run(args, verbose=False):
 116     if verbose:
 117         print("running: " + ' '.join(args))
 118     sys.stdout.flush()
 119     # Use Popen here instead of call() as it apparently allows powershell on
 120     # Windows to not lock up waiting for input presumably.
 121     ret = subprocess.Popen(args)
 122     code = ret.wait()
 123     if code != 0:
 124         err = "failed to run: " + ' '.join(args)
 125         if verbose:
 126             raise RuntimeError(err)
 127         sys.exit(err)
 128
 129 def stage0_data(rust_root):
 130     nightlies = os.path.join(rust_root, "src/stage0.txt")
 131     data = {}
 132     with open(nightlies, 'r') as nightlies:
 133         for line in nightlies:
 134             line = line.rstrip()  # Strip newline character, '\n'
 135             if line.startswith("#") or line == '':
 136                 continue
 137             a, b = line.split(": ", 1)
 138             data[a] = b
 139     return data
 140
 141 def format_build_time(duration):
 142     return str(datetime.timedelta(seconds=int(duration)))
 143
 144
 145 class RustBuild(object):
 146     def download_stage0(self):
 147         cache_dst = os.path.join(self.build_dir, "cache")
 148         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 149         cargo_cache = os.path.join(cache_dst, self.stage0_cargo_rev())
 150         if not os.path.exists(rustc_cache):
 151             os.makedirs(rustc_cache)
 152         if not os.path.exists(cargo_cache):
 153             os.makedirs(cargo_cache)
 154
 155         if self.rustc().startswith(self.bin_root()) and \
 156                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 157             self.print_what_it_means_to_bootstrap()
 158             if os.path.exists(self.bin_root()):
 159                 shutil.rmtree(self.bin_root())
 160             channel = self.stage0_rustc_channel()
 161             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 162             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 163             tarball = os.path.join(rustc_cache, filename)
 164             if not os.path.exists(tarball):
 165                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 166             unpack(tarball, self.bin_root(),
 167                    match="rust-std-" + self.build,
 168                    verbose=self.verbose)
 169
 170             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 171             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 172             tarball = os.path.join(rustc_cache, filename)
 173             if not os.path.exists(tarball):
 174                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 175             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 176             with open(self.rustc_stamp(), 'w') as f:
 177                 f.write(self.stage0_rustc_date())
 178
 179         if self.cargo().startswith(self.bin_root()) and \
 180                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 181             self.print_what_it_means_to_bootstrap()
 182             filename = "cargo-nightly-{}.tar.gz".format(self.build)
 183             url = "https://s3.amazonaws.com/rust-lang-ci/cargo-builds/" + self.stage0_cargo_rev()
 184             tarball = os.path.join(cargo_cache, filename)
 185             if not os.path.exists(tarball):
 186                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 187             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 188             with open(self.cargo_stamp(), 'w') as f:
 189                 f.write(self.stage0_cargo_rev())
 190
 191     def stage0_cargo_rev(self):
 192         return self._cargo_rev
 193
 194     def stage0_rustc_date(self):
 195         return self._rustc_date
 196
 197     def stage0_rustc_channel(self):
 198         return self._rustc_channel
 199
 200     def rustc_stamp(self):
 201         return os.path.join(self.bin_root(), '.rustc-stamp')
 202
 203     def cargo_stamp(self):
 204         return os.path.join(self.bin_root(), '.cargo-stamp')
 205
 206     def rustc_out_of_date(self):
 207         if not os.path.exists(self.rustc_stamp()) or self.clean:
 208             return True
 209         with open(self.rustc_stamp(), 'r') as f:
 210             return self.stage0_rustc_date() != f.read()
 211
 212     def cargo_out_of_date(self):
 213         if not os.path.exists(self.cargo_stamp()) or self.clean:
 214             return True
 215         with open(self.cargo_stamp(), 'r') as f:
 216             return self.stage0_cargo_rev() != f.read()
 217
 218     def bin_root(self):
 219         return os.path.join(self.build_dir, self.build, "stage0")
 220
 221     def get_toml(self, key):
 222         for line in self.config_toml.splitlines():
 223             if line.startswith(key + ' ='):
 224                 return self.get_string(line)
 225         return None
 226
 227     def get_mk(self, key):
 228         for line in iter(self.config_mk.splitlines()):
 229             if line.startswith(key):
 230                 return line[line.find(':=') + 2:].strip()
 231         return None
 232
 233     def cargo(self):
 234         config = self.get_toml('cargo')
 235         if config:
 236             return config
 237         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 238         if config:
 239             return config + '/bin/cargo' + self.exe_suffix()
 240         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 241
 242     def rustc(self):
 243         config = self.get_toml('rustc')
 244         if config:
 245             return config
 246         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 247         if config:
 248             return config + '/bin/rustc' + self.exe_suffix()
 249         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 250
 251     def get_string(self, line):
 252         start = line.find('"')
 253         end = start + 1 + line[start + 1:].find('"')
 254         return line[start + 1:end]
 255
 256     def exe_suffix(self):
 257         if sys.platform == 'win32':
 258             return '.exe'
 259         else:
 260             return ''
 261
 262     def print_what_it_means_to_bootstrap(self):
 263         if hasattr(self, 'printed'):
 264             return
 265         self.printed = True
 266         if os.path.exists(self.bootstrap_binary()):
 267             return
 268         if not '--help' in sys.argv or len(sys.argv) == 1:
 269             return
 270
 271         print('info: the build system for Rust is written in Rust, so this')
 272         print('      script is now going to download a stage0 rust compiler')
 273         print('      and then compile the build system itself')
 274         print('')
 275         print('info: in the meantime you can read more about rustbuild at')
 276         print('      src/bootstrap/README.md before the download finishes')
 277
 278     def bootstrap_binary(self):
 279         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
 280
 281     def build_bootstrap(self):
 282         self.print_what_it_means_to_bootstrap()
 283         build_dir = os.path.join(self.build_dir, "bootstrap")
 284         if self.clean and os.path.exists(build_dir):
 285             shutil.rmtree(build_dir)
 286         env = os.environ.copy()
 287         env["CARGO_TARGET_DIR"] = build_dir
 288         env["RUSTC"] = self.rustc()
 289         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 290         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 291         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 292                       os.pathsep + env["PATH"]
 293         if not os.path.isfile(self.cargo()):
 294             raise Exception("no cargo executable found at `%s`" % self.cargo())
 295         args = [self.cargo(), "build", "--manifest-path",
 296                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 297         if self.use_vendored_sources:
 298             args.append("--frozen")
 299         self.run(args, env)
 300
 301     def run(self, args, env):
 302         proc = subprocess.Popen(args, env=env)
 303         ret = proc.wait()
 304         if ret != 0:
 305             sys.exit(ret)
 306
 307     def build_triple(self):
 308         default_encoding = sys.getdefaultencoding()
 309         config = self.get_toml('build')
 310         if config:
 311             return config
 312         config = self.get_mk('CFG_BUILD')
 313         if config:
 314             return config
 315         try:
 316             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 317             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 318         except (subprocess.CalledProcessError, WindowsError):
 319             if sys.platform == 'win32':
 320                 return 'x86_64-pc-windows-msvc'
 321             err = "uname not found"
 322             if self.verbose:
 323                 raise Exception(err)
 324             sys.exit(err)
 325
 326         # Darwin's `uname -s` lies and always returns i386. We have to use
 327         # sysctl instead.
 328         if ostype == 'Darwin' and cputype == 'i686':
 329             args = ['sysctl', 'hw.optional.x86_64']
 330             sysctl = subprocess.check_output(args).decode(default_encoding)
 331             if ': 1' in sysctl:
 332                 cputype = 'x86_64'
 333
 334         # The goal here is to come up with the same triple as LLVM would,
 335         # at least for the subset of platforms we're willing to target.
 336         if ostype == 'Linux':
 337             ostype = 'unknown-linux-gnu'
 338         elif ostype == 'FreeBSD':
 339             ostype = 'unknown-freebsd'
 340         elif ostype == 'DragonFly':
 341             ostype = 'unknown-dragonfly'
 342         elif ostype == 'Bitrig':
 343             ostype = 'unknown-bitrig'
 344         elif ostype == 'OpenBSD':
 345             ostype = 'unknown-openbsd'
 346         elif ostype == 'NetBSD':
 347             ostype = 'unknown-netbsd'
 348         elif ostype == 'Darwin':
 349             ostype = 'apple-darwin'
 350         elif ostype.startswith('MINGW'):
 351             # msys' `uname` does not print gcc configuration, but prints msys
 352             # configuration. so we cannot believe `uname -m`:
 353             # msys1 is always i686 and msys2 is always x86_64.
 354             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 355             # MINGW64 on x86_64.
 356             ostype = 'pc-windows-gnu'
 357             cputype = 'i686'
 358             if os.environ.get('MSYSTEM') == 'MINGW64':
 359                 cputype = 'x86_64'
 360         elif ostype.startswith('MSYS'):
 361             ostype = 'pc-windows-gnu'
 362         elif ostype.startswith('CYGWIN_NT'):
 363             cputype = 'i686'
 364             if ostype.endswith('WOW64'):
 365                 cputype = 'x86_64'
 366             ostype = 'pc-windows-gnu'
 367         else:
 368             err = "unknown OS type: " + ostype
 369             if self.verbose:
 370                 raise ValueError(err)
 371             sys.exit(err)
 372
 373         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 374             cputype = 'i686'
 375         elif cputype in {'xscale', 'arm'}:
 376             cputype = 'arm'
 377         elif cputype == 'armv7l':
 378             cputype = 'arm'
 379             ostype += 'eabihf'
 380         elif cputype == 'aarch64':
 381             cputype = 'aarch64'
 382         elif cputype == 'mips':
 383             if sys.byteorder == 'big':
 384                 cputype = 'mips'
 385             elif sys.byteorder == 'little':
 386                 cputype = 'mipsel'
 387             else:
 388                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 389         elif cputype == 'mips64':
 390             if sys.byteorder == 'big':
 391                 cputype = 'mips64'
 392             elif sys.byteorder == 'little':
 393                 cputype = 'mips64el'
 394             else:
 395                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 396             # only the n64 ABI is supported, indicate it
 397             ostype += 'abi64'
 398         elif cputype in {'powerpc', 'ppc', 'ppc64'}:
 399             cputype = 'powerpc'
 400         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 401             cputype = 'x86_64'
 402         else:
 403             err = "unknown cpu type: " + cputype
 404             if self.verbose:
 405                 raise ValueError(err)
 406             sys.exit(err)
 407
 408         return "{}-{}".format(cputype, ostype)
 409
 410 def main():
 411     parser = argparse.ArgumentParser(description='Build rust')
 412     parser.add_argument('--config')
 413     parser.add_argument('--clean', action='store_true')
 414     parser.add_argument('-v', '--verbose', action='store_true')
 415
 416     args = [a for a in sys.argv if a != '-h' and a != '--help']
 417     args, _ = parser.parse_known_args(args)
 418
 419     # Configure initial bootstrap
 420     rb = RustBuild()
 421     rb.config_toml = ''
 422     rb.config_mk = ''
 423     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 424     rb.build_dir = os.path.join(os.getcwd(), "build")
 425     rb.verbose = args.verbose
 426     rb.clean = args.clean
 427
 428     try:
 429         with open(args.config or 'config.toml') as config:
 430             rb.config_toml = config.read()
 431     except:
 432         pass
 433     try:
 434         rb.config_mk = open('config.mk').read()
 435     except:
 436         pass
 437
 438     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
 439                               'CFG_ENABLE_VENDOR' in rb.config_mk
 440
 441     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
 442         if os.environ.get('USER') != os.environ['SUDO_USER']:
 443             rb.use_vendored_sources = True
 444             print('info: looks like you are running this command under `sudo`')
 445             print('      and so in order to preserve your $HOME this will now')
 446             print('      use vendored sources by default. Note that if this')
 447             print('      does not work you should run a normal build first')
 448             print('      before running a command like `sudo make intall`')
 449
 450     if rb.use_vendored_sources:
 451         if not os.path.exists('.cargo'):
 452             os.makedirs('.cargo')
 453         with open('.cargo/config','w') as f:
 454             f.write("""
 455                 [source.crates-io]
 456                 replace-with = 'vendored-sources'
 457                 registry = 'https://example.com'
 458
 459                 [source.vendored-sources]
 460                 directory = '{}/src/vendor'
 461             """.format(rb.rust_root))
 462     else:
 463         if os.path.exists('.cargo'):
 464             shutil.rmtree('.cargo')
 465
 466     data = stage0_data(rb.rust_root)
 467     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 468     rb._cargo_rev = data['cargo']
 469
 470     start_time = time()
 471
 472     # Fetch/build the bootstrap
 473     rb.build = rb.build_triple()
 474     rb.download_stage0()
 475     sys.stdout.flush()
 476     rb.build_bootstrap()
 477     sys.stdout.flush()
 478
 479     # Run the bootstrap
 480     args = [rb.bootstrap_binary()]
 481     args.extend(sys.argv[1:])
 482     env = os.environ.copy()
 483     env["BUILD"] = rb.build
 484     env["SRC"] = rb.rust_root
 485     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 486     rb.run(args, env)
 487
 488     end_time = time()
 489
 490     print("Build completed in %s" % format_build_time(end_time - start_time))
 491
 492 if __name__ == '__main__':
 493     main()