1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__ import print_function
26 def get(url, path, verbose=False):
27 sha_url = url + ".sha256"
28 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
29 temp_path = temp_file.name
30 with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
31 sha_path = sha_file.name
34 download(sha_path, sha_url, False, verbose)
35 if os.path.exists(path):
36 if verify(path, sha_path, False):
38 print("using already-download file " + path)
42 print("ignoring already-download file " + path + " due to failed verification")
44 download(temp_path, url, True, verbose)
45 if not verify(temp_path, sha_path, verbose):
46 raise RuntimeError("failed verification")
48 print("moving {} to {}".format(temp_path, path))
49 shutil.move(temp_path, path)
51 delete_if_present(sha_path, verbose)
52 delete_if_present(temp_path, verbose)
55 def delete_if_present(path, verbose):
56 if os.path.isfile(path):
58 print("removing " + path)
62 def download(path, url, probably_big, verbose):
65 _download(path, url, probably_big, verbose, True)
68 print("\nspurious failure, trying again")
69 _download(path, url, probably_big, verbose, False)
72 def _download(path, url, probably_big, verbose, exception):
73 if probably_big or verbose:
74 print("downloading {}".format(url))
75 # see http://serverfault.com/questions/301128/how-to-download
76 if sys.platform == 'win32':
77 run(["PowerShell.exe", "/nologo", "-Command",
78 "(New-Object System.Net.WebClient)"
79 ".DownloadFile('{}', '{}')".format(url, path)],
83 if probably_big or verbose:
87 run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
92 def verify(path, sha_path, verbose):
94 print("verifying " + path)
95 with open(path, "rb") as f:
96 found = hashlib.sha256(f.read()).hexdigest()
97 with open(sha_path, "r") as f:
98 expected = f.readline().split()[0]
99 verified = found == expected
101 print("invalid checksum:\n"
103 " expected: {}".format(found, expected))
107 def unpack(tarball, dst, verbose=False, match=None):
108 print("extracting " + tarball)
109 fname = os.path.basename(tarball).replace(".tar.gz", "")
110 with contextlib.closing(tarfile.open(tarball)) as tar:
111 for p in tar.getnames():
114 name = p.replace(fname + "/", "", 1)
115 if match is not None and not name.startswith(match):
117 name = name[len(match) + 1:]
119 fp = os.path.join(dst, name)
121 print(" extracting " + p)
123 tp = os.path.join(dst, p)
124 if os.path.isdir(tp) and os.path.exists(fp):
127 shutil.rmtree(os.path.join(dst, fname))
129 def run(args, verbose=False, exception=False):
131 print("running: " + ' '.join(args))
133 # Use Popen here instead of call() as it apparently allows powershell on
134 # Windows to not lock up waiting for input presumably.
135 ret = subprocess.Popen(args)
138 err = "failed to run: " + ' '.join(args)
139 if verbose or exception:
140 raise RuntimeError(err)
143 def stage0_data(rust_root):
144 nightlies = os.path.join(rust_root, "src/stage0.txt")
146 with open(nightlies, 'r') as nightlies:
147 for line in nightlies:
148 line = line.rstrip() # Strip newline character, '\n'
149 if line.startswith("#") or line == '':
151 a, b = line.split(": ", 1)
155 def format_build_time(duration):
156 return str(datetime.timedelta(seconds=int(duration)))
159 class RustBuild(object):
160 def download_stage0(self):
161 cache_dst = os.path.join(self.build_dir, "cache")
162 rustc_cache = os.path.join(cache_dst, self.stage0_date())
163 if not os.path.exists(rustc_cache):
164 os.makedirs(rustc_cache)
166 rustc_channel = self.stage0_rustc_channel()
167 cargo_channel = self.stage0_cargo_channel()
169 if self.rustc().startswith(self.bin_root()) and \
170 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
171 self.print_what_it_means_to_bootstrap()
172 if os.path.exists(self.bin_root()):
173 shutil.rmtree(self.bin_root())
174 filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
175 url = self._download_url + "/dist/" + self.stage0_date()
176 tarball = os.path.join(rustc_cache, filename)
177 if not os.path.exists(tarball):
178 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
179 unpack(tarball, self.bin_root(),
180 match="rust-std-" + self.build,
181 verbose=self.verbose)
183 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
184 url = self._download_url + "/dist/" + self.stage0_date()
185 tarball = os.path.join(rustc_cache, filename)
186 if not os.path.exists(tarball):
187 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
188 unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
189 self.fix_executable(self.bin_root() + "/bin/rustc")
190 self.fix_executable(self.bin_root() + "/bin/rustdoc")
191 with open(self.rustc_stamp(), 'w') as f:
192 f.write(self.stage0_date())
194 if "pc-windows-gnu" in self.build:
195 filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
196 url = self._download_url + "/dist/" + self.stage0_date()
197 tarball = os.path.join(rustc_cache, filename)
198 if not os.path.exists(tarball):
199 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
200 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
202 if self.cargo().startswith(self.bin_root()) and \
203 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
204 self.print_what_it_means_to_bootstrap()
205 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
206 url = self._download_url + "/dist/" + self.stage0_date()
207 tarball = os.path.join(rustc_cache, filename)
208 if not os.path.exists(tarball):
209 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
210 unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
211 self.fix_executable(self.bin_root() + "/bin/cargo")
212 with open(self.cargo_stamp(), 'w') as f:
213 f.write(self.stage0_date())
215 def fix_executable(self, fname):
216 # If we're on NixOS we need to change the path to the dynamic loader
218 default_encoding = sys.getdefaultencoding()
220 ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
221 except (subprocess.CalledProcessError, WindowsError):
224 if ostype != "Linux":
227 if not os.path.exists("/etc/NIXOS"):
229 if os.path.exists("/lib"):
232 # At this point we're pretty sure the user is running NixOS
233 print("info: you seem to be running NixOS. Attempting to patch " + fname)
236 interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
237 interpreter = interpreter.strip().decode(default_encoding)
238 except subprocess.CalledProcessError as e:
239 print("warning: failed to call patchelf: %s" % e)
242 loader = interpreter.split("/")[-1]
245 ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
246 ldd_output = ldd_output.strip().decode(default_encoding)
247 except subprocess.CalledProcessError as e:
248 print("warning: unable to call ldd: %s" % e)
251 for line in ldd_output.splitlines():
252 libname = line.split()[0]
253 if libname.endswith(loader):
254 loader_path = libname[:len(libname) - len(loader)]
257 print("warning: unable to find the path to the dynamic linker")
260 correct_interpreter = loader_path + loader
263 subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
264 except subprocess.CalledProcessError as e:
265 print("warning: failed to call patchelf: %s" % e)
268 def stage0_date(self):
271 def stage0_rustc_channel(self):
272 return self._rustc_channel
274 def stage0_cargo_channel(self):
275 return self._cargo_channel
277 def rustc_stamp(self):
278 return os.path.join(self.bin_root(), '.rustc-stamp')
280 def cargo_stamp(self):
281 return os.path.join(self.bin_root(), '.cargo-stamp')
283 def rustc_out_of_date(self):
284 if not os.path.exists(self.rustc_stamp()) or self.clean:
286 with open(self.rustc_stamp(), 'r') as f:
287 return self.stage0_date() != f.read()
289 def cargo_out_of_date(self):
290 if not os.path.exists(self.cargo_stamp()) or self.clean:
292 with open(self.cargo_stamp(), 'r') as f:
293 return self.stage0_date() != f.read()
296 return os.path.join(self.build_dir, self.build, "stage0")
298 def get_toml(self, key):
299 for line in self.config_toml.splitlines():
300 if line.startswith(key + ' ='):
301 return self.get_string(line)
304 def get_mk(self, key):
305 for line in iter(self.config_mk.splitlines()):
306 if line.startswith(key + ' '):
307 var = line[line.find(':=') + 2:].strip()
313 config = self.get_toml('cargo')
316 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
318 return config + '/bin/cargo' + self.exe_suffix()
319 return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
322 config = self.get_toml('rustc')
325 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
327 return config + '/bin/rustc' + self.exe_suffix()
328 return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
330 def get_string(self, line):
331 start = line.find('"')
332 end = start + 1 + line[start + 1:].find('"')
333 return line[start + 1:end]
335 def exe_suffix(self):
336 if sys.platform == 'win32':
341 def print_what_it_means_to_bootstrap(self):
342 if hasattr(self, 'printed'):
345 if os.path.exists(self.bootstrap_binary()):
347 if not '--help' in sys.argv or len(sys.argv) == 1:
350 print('info: the build system for Rust is written in Rust, so this')
351 print(' script is now going to download a stage0 rust compiler')
352 print(' and then compile the build system itself')
354 print('info: in the meantime you can read more about rustbuild at')
355 print(' src/bootstrap/README.md before the download finishes')
357 def bootstrap_binary(self):
358 return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
360 def build_bootstrap(self):
361 self.print_what_it_means_to_bootstrap()
362 build_dir = os.path.join(self.build_dir, "bootstrap")
363 if self.clean and os.path.exists(build_dir):
364 shutil.rmtree(build_dir)
365 env = os.environ.copy()
366 env["CARGO_TARGET_DIR"] = build_dir
367 env["RUSTC"] = self.rustc()
368 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
369 (os.pathsep + env["LD_LIBRARY_PATH"]) \
370 if "LD_LIBRARY_PATH" in env else ""
371 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
372 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
373 if "DYLD_LIBRARY_PATH" in env else ""
374 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
375 (os.pathsep + env["LIBRARY_PATH"]) \
376 if "LIBRARY_PATH" in env else ""
377 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
378 os.pathsep + env["PATH"]
379 if not os.path.isfile(self.cargo()):
380 raise Exception("no cargo executable found at `%s`" % self.cargo())
381 args = [self.cargo(), "build", "--manifest-path",
382 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
383 if self.use_locked_deps:
384 args.append("--locked")
385 if self.use_vendored_sources:
386 args.append("--frozen")
389 def run(self, args, env):
390 proc = subprocess.Popen(args, env=env)
395 def build_triple(self):
396 default_encoding = sys.getdefaultencoding()
397 config = self.get_toml('build')
400 config = self.get_mk('CFG_BUILD')
404 ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
405 cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
406 except (subprocess.CalledProcessError, OSError):
407 if sys.platform == 'win32':
408 return 'x86_64-pc-windows-msvc'
409 err = "uname not found"
414 # The goal here is to come up with the same triple as LLVM would,
415 # at least for the subset of platforms we're willing to target.
416 if ostype == 'Linux':
417 os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
418 if os_from_sp == 'Android':
419 ostype = 'linux-android'
421 ostype = 'unknown-linux-gnu'
422 elif ostype == 'FreeBSD':
423 ostype = 'unknown-freebsd'
424 elif ostype == 'DragonFly':
425 ostype = 'unknown-dragonfly'
426 elif ostype == 'Bitrig':
427 ostype = 'unknown-bitrig'
428 elif ostype == 'OpenBSD':
429 ostype = 'unknown-openbsd'
430 elif ostype == 'NetBSD':
431 ostype = 'unknown-netbsd'
432 elif ostype == 'SunOS':
433 ostype = 'sun-solaris'
434 # On Solaris, uname -m will return a machine classification instead
435 # of a cpu type, so uname -p is recommended instead. However, the
436 # output from that option is too generic for our purposes (it will
437 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
438 # must be used instead.
440 cputype = subprocess.check_output(['isainfo',
441 '-k']).strip().decode(default_encoding)
442 except (subprocess.CalledProcessError, OSError):
443 err = "isainfo not found"
447 elif ostype == 'Darwin':
448 ostype = 'apple-darwin'
449 elif ostype == 'Haiku':
450 ostype = 'unknown-haiku'
451 elif ostype.startswith('MINGW'):
452 # msys' `uname` does not print gcc configuration, but prints msys
453 # configuration. so we cannot believe `uname -m`:
454 # msys1 is always i686 and msys2 is always x86_64.
455 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
457 ostype = 'pc-windows-gnu'
459 if os.environ.get('MSYSTEM') == 'MINGW64':
461 elif ostype.startswith('MSYS'):
462 ostype = 'pc-windows-gnu'
463 elif ostype.startswith('CYGWIN_NT'):
465 if ostype.endswith('WOW64'):
467 ostype = 'pc-windows-gnu'
469 err = "unknown OS type: " + ostype
471 raise ValueError(err)
474 if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
476 elif cputype in {'xscale', 'arm'}:
478 if ostype == 'linux-android':
479 ostype = 'linux-androideabi'
480 elif cputype == 'armv6l':
482 if ostype == 'linux-android':
483 ostype = 'linux-androideabi'
486 elif cputype in {'armv7l', 'armv8l'}:
488 if ostype == 'linux-android':
489 ostype = 'linux-androideabi'
492 elif cputype in {'aarch64', 'arm64'}:
494 elif cputype == 'mips':
495 if sys.byteorder == 'big':
497 elif sys.byteorder == 'little':
500 raise ValueError('unknown byteorder: ' + sys.byteorder)
501 elif cputype == 'mips64':
502 if sys.byteorder == 'big':
504 elif sys.byteorder == 'little':
507 raise ValueError('unknown byteorder: ' + sys.byteorder)
508 # only the n64 ABI is supported, indicate it
510 elif cputype in {'powerpc', 'ppc'}:
512 elif cputype in {'powerpc64', 'ppc64'}:
513 cputype = 'powerpc64'
514 elif cputype in {'powerpc64le', 'ppc64le'}:
515 cputype = 'powerpc64le'
516 elif cputype == 'sparcv9':
518 elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
520 elif cputype == 's390x':
522 elif cputype == 'BePC':
525 err = "unknown cpu type: " + cputype
527 raise ValueError(err)
530 return "{}-{}".format(cputype, ostype)
533 parser = argparse.ArgumentParser(description='Build rust')
534 parser.add_argument('--config')
535 parser.add_argument('--clean', action='store_true')
536 parser.add_argument('-v', '--verbose', action='store_true')
538 args = [a for a in sys.argv if a != '-h' and a != '--help']
539 args, _ = parser.parse_known_args(args)
541 # Configure initial bootstrap
545 rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
546 rb.build_dir = os.path.join(os.getcwd(), "build")
547 rb.verbose = args.verbose
548 rb.clean = args.clean
551 with open(args.config or 'config.toml') as config:
552 rb.config_toml = config.read()
556 rb.config_mk = open('config.mk').read()
560 rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
561 'CFG_ENABLE_VENDOR' in rb.config_mk
563 rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
564 'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
566 if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
567 if os.environ.get('USER') != os.environ['SUDO_USER']:
568 rb.use_vendored_sources = True
569 print('info: looks like you are running this command under `sudo`')
570 print(' and so in order to preserve your $HOME this will now')
571 print(' use vendored sources by default. Note that if this')
572 print(' does not work you should run a normal build first')
573 print(' before running a command like `sudo make install`')
575 if rb.use_vendored_sources:
576 if not os.path.exists('.cargo'):
577 os.makedirs('.cargo')
578 with open('.cargo/config','w') as f:
581 replace-with = 'vendored-sources'
582 registry = 'https://example.com'
584 [source.vendored-sources]
585 directory = '{}/src/vendor'
586 """.format(rb.rust_root))
588 if os.path.exists('.cargo'):
589 shutil.rmtree('.cargo')
591 data = stage0_data(rb.rust_root)
592 rb._date = data['date']
593 rb._rustc_channel = data['rustc']
594 rb._cargo_channel = data['cargo']
596 rb._download_url = 'https://dev-static.rust-lang.org'
598 rb._download_url = 'https://static.rust-lang.org'
600 # Fetch/build the bootstrap
601 rb.build = rb.build_triple()
608 args = [rb.bootstrap_binary()]
609 args.extend(sys.argv[1:])
610 env = os.environ.copy()
611 env["BUILD"] = rb.build
612 env["SRC"] = rb.rust_root
613 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
618 help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
621 if not help_triggered:
622 print("Build completed successfully in %s" % format_build_time(time() - start_time))
623 except (SystemExit, KeyboardInterrupt) as e:
624 if hasattr(e, 'code') and isinstance(e.code, int):
629 if not help_triggered:
630 print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
633 if __name__ == '__main__':