src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import argparse
  12 import contextlib
  13 import datetime
  14 import hashlib
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tarfile
  20 import tempfile
  21
  22 from time import time
  23
  24
  25 def get(url, path, verbose=False):
  26     sha_url = url + ".sha256"
  27     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  28         temp_path = temp_file.name
  29     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  30         sha_path = sha_file.name
  31
  32     try:
  33         download(sha_path, sha_url, verbose)
  34         if os.path.exists(path):
  35             if verify(path, sha_path, False):
  36                 print("using already-download file " + path)
  37                 return
  38             else:
  39                 print("ignoring already-download file " + path + " due to failed verification")
  40                 os.unlink(path)
  41         download(temp_path, url, verbose)
  42         if not verify(temp_path, sha_path, True):
  43             raise RuntimeError("failed verification")
  44         print("moving {} to {}".format(temp_path, path))
  45         shutil.move(temp_path, path)
  46     finally:
  47         delete_if_present(sha_path)
  48         delete_if_present(temp_path)
  49
  50
  51 def delete_if_present(path):
  52     if os.path.isfile(path):
  53         print("removing " + path)
  54         os.unlink(path)
  55
  56
  57 def download(path, url, verbose):
  58     print("downloading {} to {}".format(url, path))
  59     # see http://serverfault.com/questions/301128/how-to-download
  60     if sys.platform == 'win32':
  61         run(["PowerShell.exe", "/nologo", "-Command",
  62              "(New-Object System.Net.WebClient)"
  63              ".DownloadFile('{}', '{}')".format(url, path)],
  64             verbose=verbose)
  65     else:
  66         run(["curl", "-o", path, url], verbose=verbose)
  67
  68
  69 def verify(path, sha_path, verbose):
  70     print("verifying " + path)
  71     with open(path, "rb") as f:
  72         found = hashlib.sha256(f.read()).hexdigest()
  73     with open(sha_path, "r") as f:
  74         expected, _ = f.readline().split()
  75     verified = found == expected
  76     if not verified and verbose:
  77         print("invalid checksum:\n"
  78                "    found:    {}\n"
  79                "    expected: {}".format(found, expected))
  80     return verified
  81
  82
  83 def unpack(tarball, dst, verbose=False, match=None):
  84     print("extracting " + tarball)
  85     fname = os.path.basename(tarball).replace(".tar.gz", "")
  86     with contextlib.closing(tarfile.open(tarball)) as tar:
  87         for p in tar.getnames():
  88             if "/" not in p:
  89                 continue
  90             name = p.replace(fname + "/", "", 1)
  91             if match is not None and not name.startswith(match):
  92                 continue
  93             name = name[len(match) + 1:]
  94
  95             fp = os.path.join(dst, name)
  96             if verbose:
  97                 print("  extracting " + p)
  98             tar.extract(p, dst)
  99             tp = os.path.join(dst, p)
 100             if os.path.isdir(tp) and os.path.exists(fp):
 101                 continue
 102             shutil.move(tp, fp)
 103     shutil.rmtree(os.path.join(dst, fname))
 104
 105 def run(args, verbose=False):
 106     if verbose:
 107         print("running: " + ' '.join(args))
 108     sys.stdout.flush()
 109     # Use Popen here instead of call() as it apparently allows powershell on
 110     # Windows to not lock up waiting for input presumably.
 111     ret = subprocess.Popen(args)
 112     code = ret.wait()
 113     if code != 0:
 114         err = "failed to run: " + ' '.join(args)
 115         if verbose:
 116             raise RuntimeError(err)
 117         sys.exit(err)
 118
 119 def stage0_data(rust_root):
 120     nightlies = os.path.join(rust_root, "src/stage0.txt")
 121     data = {}
 122     with open(nightlies, 'r') as nightlies:
 123         for line in nightlies:
 124             line = line.rstrip()  # Strip newline character, '\n'
 125             if line.startswith("#") or line == '':
 126                 continue
 127             a, b = line.split(": ", 1)
 128             data[a] = b
 129     return data
 130
 131 def format_build_time(duration):
 132     return str(datetime.timedelta(seconds=int(duration)))
 133
 134
 135 class RustBuild(object):
 136     def download_stage0(self):
 137         cache_dst = os.path.join(self.build_dir, "cache")
 138         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 139         cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
 140         if not os.path.exists(rustc_cache):
 141             os.makedirs(rustc_cache)
 142         if not os.path.exists(cargo_cache):
 143             os.makedirs(cargo_cache)
 144
 145         if self.rustc().startswith(self.bin_root()) and \
 146                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 147             if os.path.exists(self.bin_root()):
 148                 shutil.rmtree(self.bin_root())
 149             channel = self.stage0_rustc_channel()
 150             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 151             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 152             tarball = os.path.join(rustc_cache, filename)
 153             if not os.path.exists(tarball):
 154                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 155             unpack(tarball, self.bin_root(),
 156                    match="rust-std-" + self.build,
 157                    verbose=self.verbose)
 158
 159             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 160             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 161             tarball = os.path.join(rustc_cache, filename)
 162             if not os.path.exists(tarball):
 163                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 164             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 165             with open(self.rustc_stamp(), 'w') as f:
 166                 f.write(self.stage0_rustc_date())
 167
 168         if self.cargo().startswith(self.bin_root()) and \
 169                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 170             channel = self.stage0_cargo_channel()
 171             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
 172             url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
 173             tarball = os.path.join(cargo_cache, filename)
 174             if not os.path.exists(tarball):
 175                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 176             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 177             with open(self.cargo_stamp(), 'w') as f:
 178                 f.write(self.stage0_cargo_date())
 179
 180     def stage0_cargo_date(self):
 181         return self._cargo_date
 182
 183     def stage0_cargo_channel(self):
 184         return self._cargo_channel
 185
 186     def stage0_rustc_date(self):
 187         return self._rustc_date
 188
 189     def stage0_rustc_channel(self):
 190         return self._rustc_channel
 191
 192     def rustc_stamp(self):
 193         return os.path.join(self.bin_root(), '.rustc-stamp')
 194
 195     def cargo_stamp(self):
 196         return os.path.join(self.bin_root(), '.cargo-stamp')
 197
 198     def rustc_out_of_date(self):
 199         if not os.path.exists(self.rustc_stamp()) or self.clean:
 200             return True
 201         with open(self.rustc_stamp(), 'r') as f:
 202             return self.stage0_rustc_date() != f.read()
 203
 204     def cargo_out_of_date(self):
 205         if not os.path.exists(self.cargo_stamp()) or self.clean:
 206             return True
 207         with open(self.cargo_stamp(), 'r') as f:
 208             return self.stage0_cargo_date() != f.read()
 209
 210     def bin_root(self):
 211         return os.path.join(self.build_dir, self.build, "stage0")
 212
 213     def get_toml(self, key):
 214         for line in self.config_toml.splitlines():
 215             if line.startswith(key + ' ='):
 216                 return self.get_string(line)
 217         return None
 218
 219     def get_mk(self, key):
 220         for line in iter(self.config_mk.splitlines()):
 221             if line.startswith(key):
 222                 return line[line.find(':=') + 2:].strip()
 223         return None
 224
 225     def cargo(self):
 226         config = self.get_toml('cargo')
 227         if config:
 228             return config
 229         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 230
 231     def rustc(self):
 232         config = self.get_toml('rustc')
 233         if config:
 234             return config
 235         config = self.get_mk('CFG_LOCAL_RUST')
 236         if config:
 237             return config + '/bin/rustc' + self.exe_suffix()
 238         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 239
 240     def get_string(self, line):
 241         start = line.find('"')
 242         end = start + 1 + line[start + 1:].find('"')
 243         return line[start + 1:end]
 244
 245     def exe_suffix(self):
 246         if sys.platform == 'win32':
 247             return '.exe'
 248         else:
 249             return ''
 250
 251     def build_bootstrap(self):
 252         build_dir = os.path.join(self.build_dir, "bootstrap")
 253         if self.clean and os.path.exists(build_dir):
 254             shutil.rmtree(build_dir)
 255         env = os.environ.copy()
 256         env["CARGO_TARGET_DIR"] = build_dir
 257         env["RUSTC"] = self.rustc()
 258         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 259         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 260         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 261                       os.pathsep + env["PATH"]
 262         args = [self.cargo(), "build", "--manifest-path",
 263                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 264         if self.use_vendored_sources:
 265             args.append("--frozen")
 266         self.run(args, env)
 267
 268     def run(self, args, env):
 269         proc = subprocess.Popen(args, env=env)
 270         ret = proc.wait()
 271         if ret != 0:
 272             sys.exit(ret)
 273
 274     def build_triple(self):
 275         default_encoding = sys.getdefaultencoding()
 276         config = self.get_toml('build')
 277         if config:
 278             return config
 279         config = self.get_mk('CFG_BUILD')
 280         if config:
 281             return config
 282         try:
 283             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 284             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 285         except (subprocess.CalledProcessError, WindowsError):
 286             if sys.platform == 'win32':
 287                 return 'x86_64-pc-windows-msvc'
 288             err = "uname not found"
 289             if self.verbose:
 290                 raise Exception(err)
 291             sys.exit(err)
 292
 293         # Darwin's `uname -s` lies and always returns i386. We have to use
 294         # sysctl instead.
 295         if ostype == 'Darwin' and cputype == 'i686':
 296             args = ['sysctl', 'hw.optional.x86_64']
 297             sysctl = subprocess.check_output(args).decode(default_encoding)
 298             if ': 1' in sysctl:
 299                 cputype = 'x86_64'
 300
 301         # The goal here is to come up with the same triple as LLVM would,
 302         # at least for the subset of platforms we're willing to target.
 303         if ostype == 'Linux':
 304             ostype = 'unknown-linux-gnu'
 305         elif ostype == 'FreeBSD':
 306             ostype = 'unknown-freebsd'
 307         elif ostype == 'DragonFly':
 308             ostype = 'unknown-dragonfly'
 309         elif ostype == 'Bitrig':
 310             ostype = 'unknown-bitrig'
 311         elif ostype == 'OpenBSD':
 312             ostype = 'unknown-openbsd'
 313         elif ostype == 'NetBSD':
 314             ostype = 'unknown-netbsd'
 315         elif ostype == 'Darwin':
 316             ostype = 'apple-darwin'
 317         elif ostype.startswith('MINGW'):
 318             # msys' `uname` does not print gcc configuration, but prints msys
 319             # configuration. so we cannot believe `uname -m`:
 320             # msys1 is always i686 and msys2 is always x86_64.
 321             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 322             # MINGW64 on x86_64.
 323             ostype = 'pc-windows-gnu'
 324             cputype = 'i686'
 325             if os.environ.get('MSYSTEM') == 'MINGW64':
 326                 cputype = 'x86_64'
 327         elif ostype.startswith('MSYS'):
 328             ostype = 'pc-windows-gnu'
 329         elif ostype.startswith('CYGWIN_NT'):
 330             cputype = 'i686'
 331             if ostype.endswith('WOW64'):
 332                 cputype = 'x86_64'
 333             ostype = 'pc-windows-gnu'
 334         else:
 335             err = "unknown OS type: " + ostype
 336             if self.verbose:
 337                 raise ValueError(err)
 338             sys.exit(err)
 339
 340         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 341             cputype = 'i686'
 342         elif cputype in {'xscale', 'arm'}:
 343             cputype = 'arm'
 344         elif cputype == 'armv7l':
 345             cputype = 'arm'
 346             ostype += 'eabihf'
 347         elif cputype == 'aarch64':
 348             cputype = 'aarch64'
 349         elif cputype in {'powerpc', 'ppc', 'ppc64'}:
 350             cputype = 'powerpc'
 351         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 352             cputype = 'x86_64'
 353         else:
 354             err = "unknown cpu type: " + cputype
 355             if self.verbose:
 356                 raise ValueError(err)
 357             sys.exit(err)
 358
 359         return "{}-{}".format(cputype, ostype)
 360
 361 def main():
 362     parser = argparse.ArgumentParser(description='Build rust')
 363     parser.add_argument('--config')
 364     parser.add_argument('--clean', action='store_true')
 365     parser.add_argument('-v', '--verbose', action='store_true')
 366
 367     args = [a for a in sys.argv if a != '-h' and a != '--help']
 368     args, _ = parser.parse_known_args(args)
 369
 370     # Configure initial bootstrap
 371     rb = RustBuild()
 372     rb.config_toml = ''
 373     rb.config_mk = ''
 374     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 375     rb.build_dir = os.path.join(os.getcwd(), "build")
 376     rb.verbose = args.verbose
 377     rb.clean = args.clean
 378
 379     try:
 380         with open(args.config or 'config.toml') as config:
 381             rb.config_toml = config.read()
 382     except:
 383         pass
 384     try:
 385         rb.config_mk = open('config.mk').read()
 386     except:
 387         pass
 388
 389     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
 390                               'CFG_ENABLE_VENDOR' in rb.config_mk
 391
 392     if rb.use_vendored_sources:
 393         if not os.path.exists('.cargo'):
 394             os.makedirs('.cargo')
 395         f = open('.cargo/config','w')
 396         f.write("""
 397             [source.crates-io]
 398             replace-with = 'vendored-sources'
 399             registry = 'https://example.com'
 400
 401             [source.vendored-sources]
 402             directory = '{}/src/vendor'
 403         """.format(rb.rust_root))
 404         f.close()
 405     else:
 406         if os.path.exists('.cargo'):
 407             shutil.rmtree('.cargo')
 408     data = stage0_data(rb.rust_root)
 409     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 410     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
 411
 412     start_time = time()
 413
 414     # Fetch/build the bootstrap
 415     rb.build = rb.build_triple()
 416     rb.download_stage0()
 417     sys.stdout.flush()
 418     rb.build_bootstrap()
 419     sys.stdout.flush()
 420
 421     # Run the bootstrap
 422     args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
 423     args.extend(sys.argv[1:])
 424     env = os.environ.copy()
 425     env["BUILD"] = rb.build
 426     env["SRC"] = rb.rust_root
 427     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 428     rb.run(args, env)
 429
 430     end_time = time()
 431
 432     print("Build completed in %s" % format_build_time(end_time - start_time))
 433
 434 if __name__ == '__main__':
 435     main()