src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import argparse
  12 import contextlib
  13 import datetime
  14 import hashlib
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tarfile
  20 import tempfile
  21
  22 from time import time
  23
  24
  25 def get(url, path, verbose=False):
  26     sha_url = url + ".sha256"
  27     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  28         temp_path = temp_file.name
  29     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  30         sha_path = sha_file.name
  31
  32     try:
  33         download(sha_path, sha_url, verbose)
  34         if os.path.exists(path):
  35             if verify(path, sha_path, False):
  36                 print("using already-download file " + path)
  37                 return
  38             else:
  39                 print("ignoring already-download file " + path + " due to failed verification")
  40                 os.unlink(path)
  41         download(temp_path, url, verbose)
  42         if not verify(temp_path, sha_path, True):
  43             raise RuntimeError("failed verification")
  44         print("moving {} to {}".format(temp_path, path))
  45         shutil.move(temp_path, path)
  46     finally:
  47         delete_if_present(sha_path)
  48         delete_if_present(temp_path)
  49
  50
  51 def delete_if_present(path):
  52     if os.path.isfile(path):
  53         print("removing " + path)
  54         os.unlink(path)
  55
  56
  57 def download(path, url, verbose):
  58     print("downloading {} to {}".format(url, path))
  59     # see http://serverfault.com/questions/301128/how-to-download
  60     if sys.platform == 'win32':
  61         run(["PowerShell.exe", "/nologo", "-Command",
  62              "(New-Object System.Net.WebClient)"
  63              ".DownloadFile('{}', '{}')".format(url, path)],
  64             verbose=verbose)
  65     else:
  66         run(["curl", "-o", path, url], verbose=verbose)
  67
  68
  69 def verify(path, sha_path, verbose):
  70     print("verifying " + path)
  71     with open(path, "rb") as f:
  72         found = hashlib.sha256(f.read()).hexdigest()
  73     with open(sha_path, "r") as f:
  74         expected, _ = f.readline().split()
  75     verified = found == expected
  76     if not verified and verbose:
  77         print("invalid checksum:\n"
  78                "    found:    {}\n"
  79                "    expected: {}".format(found, expected))
  80     return verified
  81
  82
  83 def unpack(tarball, dst, verbose=False, match=None):
  84     print("extracting " + tarball)
  85     fname = os.path.basename(tarball).replace(".tar.gz", "")
  86     with contextlib.closing(tarfile.open(tarball)) as tar:
  87         for p in tar.getnames():
  88             if "/" not in p:
  89                 continue
  90             name = p.replace(fname + "/", "", 1)
  91             if match is not None and not name.startswith(match):
  92                 continue
  93             name = name[len(match) + 1:]
  94
  95             fp = os.path.join(dst, name)
  96             if verbose:
  97                 print("  extracting " + p)
  98             tar.extract(p, dst)
  99             tp = os.path.join(dst, p)
 100             if os.path.isdir(tp) and os.path.exists(fp):
 101                 continue
 102             shutil.move(tp, fp)
 103     shutil.rmtree(os.path.join(dst, fname))
 104
 105 def run(args, verbose=False):
 106     if verbose:
 107         print("running: " + ' '.join(args))
 108     sys.stdout.flush()
 109     # Use Popen here instead of call() as it apparently allows powershell on
 110     # Windows to not lock up waiting for input presumably.
 111     ret = subprocess.Popen(args)
 112     code = ret.wait()
 113     if code != 0:
 114         err = "failed to run: " + ' '.join(args)
 115         if verbose:
 116             raise RuntimeError(err)
 117         sys.exit(err)
 118
 119 def stage0_data(rust_root):
 120     nightlies = os.path.join(rust_root, "src/stage0.txt")
 121     data = {}
 122     with open(nightlies, 'r') as nightlies:
 123         for line in nightlies:
 124             line = line.rstrip()  # Strip newline character, '\n'
 125             if line.startswith("#") or line == '':
 126                 continue
 127             a, b = line.split(": ", 1)
 128             data[a] = b
 129     return data
 130
 131 def format_build_time(duration):
 132     return str(datetime.timedelta(seconds=int(duration)))
 133
 134
 135 class RustBuild(object):
 136     def download_stage0(self):
 137         cache_dst = os.path.join(self.build_dir, "cache")
 138         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 139         cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
 140         if not os.path.exists(rustc_cache):
 141             os.makedirs(rustc_cache)
 142         if not os.path.exists(cargo_cache):
 143             os.makedirs(cargo_cache)
 144
 145         if self.rustc().startswith(self.bin_root()) and \
 146                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 147             if os.path.exists(self.bin_root()):
 148                 shutil.rmtree(self.bin_root())
 149             channel = self.stage0_rustc_channel()
 150             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 151             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 152             tarball = os.path.join(rustc_cache, filename)
 153             if not os.path.exists(tarball):
 154                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 155             unpack(tarball, self.bin_root(),
 156                    match="rust-std-" + self.build,
 157                    verbose=self.verbose)
 158
 159             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 160             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 161             tarball = os.path.join(rustc_cache, filename)
 162             if not os.path.exists(tarball):
 163                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 164             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 165             with open(self.rustc_stamp(), 'w') as f:
 166                 f.write(self.stage0_rustc_date())
 167
 168         if self.cargo().startswith(self.bin_root()) and \
 169                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 170             channel = self.stage0_cargo_channel()
 171             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
 172             url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
 173             tarball = os.path.join(cargo_cache, filename)
 174             if not os.path.exists(tarball):
 175                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 176             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 177             with open(self.cargo_stamp(), 'w') as f:
 178                 f.write(self.stage0_cargo_date())
 179
 180     def stage0_cargo_date(self):
 181         return self._cargo_date
 182
 183     def stage0_cargo_channel(self):
 184         return self._cargo_channel
 185
 186     def stage0_rustc_date(self):
 187         return self._rustc_date
 188
 189     def stage0_rustc_channel(self):
 190         return self._rustc_channel
 191
 192     def rustc_stamp(self):
 193         return os.path.join(self.bin_root(), '.rustc-stamp')
 194
 195     def cargo_stamp(self):
 196         return os.path.join(self.bin_root(), '.cargo-stamp')
 197
 198     def rustc_out_of_date(self):
 199         if not os.path.exists(self.rustc_stamp()) or self.clean:
 200             return True
 201         with open(self.rustc_stamp(), 'r') as f:
 202             return self.stage0_rustc_date() != f.read()
 203
 204     def cargo_out_of_date(self):
 205         if not os.path.exists(self.cargo_stamp()) or self.clean:
 206             return True
 207         with open(self.cargo_stamp(), 'r') as f:
 208             return self.stage0_cargo_date() != f.read()
 209
 210     def bin_root(self):
 211         return os.path.join(self.build_dir, self.build, "stage0")
 212
 213     def get_toml(self, key):
 214         for line in self.config_toml.splitlines():
 215             if line.startswith(key + ' ='):
 216                 return self.get_string(line)
 217         return None
 218
 219     def get_mk(self, key):
 220         for line in iter(self.config_mk.splitlines()):
 221             if line.startswith(key):
 222                 return line[line.find(':=') + 2:].strip()
 223         return None
 224
 225     def cargo(self):
 226         config = self.get_toml('cargo')
 227         if config:
 228             return config
 229         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 230
 231     def rustc(self):
 232         config = self.get_toml('rustc')
 233         if config:
 234             return config
 235         config = self.get_mk('CFG_LOCAL_RUST')
 236         if config:
 237             return config + '/bin/rustc' + self.exe_suffix()
 238         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 239
 240     def get_string(self, line):
 241         start = line.find('"')
 242         end = start + 1 + line[start + 1:].find('"')
 243         return line[start + 1:end]
 244
 245     def exe_suffix(self):
 246         if sys.platform == 'win32':
 247             return '.exe'
 248         else:
 249             return ''
 250
 251     def build_bootstrap(self):
 252         build_dir = os.path.join(self.build_dir, "bootstrap")
 253         if self.clean and os.path.exists(build_dir):
 254             shutil.rmtree(build_dir)
 255         env = os.environ.copy()
 256         env["CARGO_TARGET_DIR"] = build_dir
 257         env["RUSTC"] = self.rustc()
 258         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 259         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 260         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 261                       os.pathsep + env["PATH"]
 262         self.run([self.cargo(), "build", "--manifest-path",
 263                   os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
 264                  env)
 265
 266     def run(self, args, env):
 267         proc = subprocess.Popen(args, env=env)
 268         ret = proc.wait()
 269         if ret != 0:
 270             sys.exit(ret)
 271
 272     def build_triple(self):
 273         default_encoding = sys.getdefaultencoding()
 274         config = self.get_toml('build')
 275         if config:
 276             return config
 277         config = self.get_mk('CFG_BUILD')
 278         if config:
 279             return config
 280         try:
 281             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 282             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 283         except (subprocess.CalledProcessError, WindowsError):
 284             if sys.platform == 'win32':
 285                 return 'x86_64-pc-windows-msvc'
 286             err = "uname not found"
 287             if self.verbose:
 288                 raise Exception(err)
 289             sys.exit(err)
 290
 291         # Darwin's `uname -s` lies and always returns i386. We have to use
 292         # sysctl instead.
 293         if ostype == 'Darwin' and cputype == 'i686':
 294             args = ['sysctl', 'hw.optional.x86_64']
 295             sysctl = subprocess.check_output(args).decode(default_encoding)
 296             if ': 1' in sysctl:
 297                 cputype = 'x86_64'
 298
 299         # The goal here is to come up with the same triple as LLVM would,
 300         # at least for the subset of platforms we're willing to target.
 301         if ostype == 'Linux':
 302             ostype = 'unknown-linux-gnu'
 303         elif ostype == 'FreeBSD':
 304             ostype = 'unknown-freebsd'
 305         elif ostype == 'DragonFly':
 306             ostype = 'unknown-dragonfly'
 307         elif ostype == 'Bitrig':
 308             ostype = 'unknown-bitrig'
 309         elif ostype == 'OpenBSD':
 310             ostype = 'unknown-openbsd'
 311         elif ostype == 'NetBSD':
 312             ostype = 'unknown-netbsd'
 313         elif ostype == 'Darwin':
 314             ostype = 'apple-darwin'
 315         elif ostype.startswith('MINGW'):
 316             # msys' `uname` does not print gcc configuration, but prints msys
 317             # configuration. so we cannot believe `uname -m`:
 318             # msys1 is always i686 and msys2 is always x86_64.
 319             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 320             # MINGW64 on x86_64.
 321             ostype = 'pc-windows-gnu'
 322             cputype = 'i686'
 323             if os.environ.get('MSYSTEM') == 'MINGW64':
 324                 cputype = 'x86_64'
 325         elif ostype.startswith('MSYS'):
 326             ostype = 'pc-windows-gnu'
 327         elif ostype.startswith('CYGWIN_NT'):
 328             cputype = 'i686'
 329             if ostype.endswith('WOW64'):
 330                 cputype = 'x86_64'
 331             ostype = 'pc-windows-gnu'
 332         else:
 333             err = "unknown OS type: " + ostype
 334             if self.verbose:
 335                 raise ValueError(err)
 336             sys.exit(err)
 337
 338         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 339             cputype = 'i686'
 340         elif cputype in {'xscale', 'arm'}:
 341             cputype = 'arm'
 342         elif cputype == 'armv7l':
 343             cputype = 'arm'
 344             ostype += 'eabihf'
 345         elif cputype == 'aarch64':
 346             cputype = 'aarch64'
 347         elif cputype in {'powerpc', 'ppc', 'ppc64'}:
 348             cputype = 'powerpc'
 349         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 350             cputype = 'x86_64'
 351         else:
 352             err = "unknown cpu type: " + cputype
 353             if self.verbose:
 354                 raise ValueError(err)
 355             sys.exit(err)
 356
 357         return "{}-{}".format(cputype, ostype)
 358
 359 def main():
 360     parser = argparse.ArgumentParser(description='Build rust')
 361     parser.add_argument('--config')
 362     parser.add_argument('--clean', action='store_true')
 363     parser.add_argument('-v', '--verbose', action='store_true')
 364
 365     args = [a for a in sys.argv if a != '-h' and a != '--help']
 366     args, _ = parser.parse_known_args(args)
 367
 368     # Configure initial bootstrap
 369     rb = RustBuild()
 370     rb.config_toml = ''
 371     rb.config_mk = ''
 372     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 373     rb.build_dir = os.path.join(os.getcwd(), "build")
 374     rb.verbose = args.verbose
 375     rb.clean = args.clean
 376
 377     try:
 378         with open(args.config or 'config.toml') as config:
 379             rb.config_toml = config.read()
 380     except:
 381         pass
 382     try:
 383         rb.config_mk = open('config.mk').read()
 384     except:
 385         pass
 386
 387     data = stage0_data(rb.rust_root)
 388     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 389     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
 390
 391     start_time = time()
 392
 393     # Fetch/build the bootstrap
 394     rb.build = rb.build_triple()
 395     rb.download_stage0()
 396     sys.stdout.flush()
 397     rb.build_bootstrap()
 398     sys.stdout.flush()
 399
 400     # Run the bootstrap
 401     args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
 402     args.extend(sys.argv[1:])
 403     env = os.environ.copy()
 404     env["BUILD"] = rb.build
 405     env["SRC"] = rb.rust_root
 406     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 407     rb.run(args, env)
 408
 409     end_time = time()
 410
 411     print("Build completed in %s" % format_build_time(end_time - start_time))
 412
 413 if __name__ == '__main__':
 414     main()