1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__ import print_function
27 def get(url, path, verbose=False):
28 sha_url = url + ".sha256"
29 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
30 temp_path = temp_file.name
31 with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
32 sha_path = sha_file.name
35 download(sha_path, sha_url, False, verbose)
36 if os.path.exists(path):
37 if verify(path, sha_path, False):
39 print("using already-download file " + path)
43 print("ignoring already-download file " + path + " due to failed verification")
45 download(temp_path, url, True, verbose)
46 if not verify(temp_path, sha_path, verbose):
47 raise RuntimeError("failed verification")
49 print("moving {} to {}".format(temp_path, path))
50 shutil.move(temp_path, path)
52 delete_if_present(sha_path, verbose)
53 delete_if_present(temp_path, verbose)
56 def delete_if_present(path, verbose):
57 if os.path.isfile(path):
59 print("removing " + path)
63 def download(path, url, probably_big, verbose):
66 _download(path, url, probably_big, verbose, True)
69 print("\nspurious failure, trying again")
70 _download(path, url, probably_big, verbose, False)
73 def _download(path, url, probably_big, verbose, exception):
74 if probably_big or verbose:
75 print("downloading {}".format(url))
76 # see http://serverfault.com/questions/301128/how-to-download
77 if sys.platform == 'win32':
78 run(["PowerShell.exe", "/nologo", "-Command",
79 "(New-Object System.Net.WebClient)"
80 ".DownloadFile('{}', '{}')".format(url, path)],
84 if probably_big or verbose:
88 run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
93 def verify(path, sha_path, verbose):
95 print("verifying " + path)
96 with open(path, "rb") as f:
97 found = hashlib.sha256(f.read()).hexdigest()
98 with open(sha_path, "r") as f:
99 expected = f.readline().split()[0]
100 verified = found == expected
102 print("invalid checksum:\n"
104 " expected: {}".format(found, expected))
108 def unpack(tarball, dst, verbose=False, match=None):
109 print("extracting " + tarball)
110 fname = os.path.basename(tarball).replace(".tar.gz", "")
111 with contextlib.closing(tarfile.open(tarball)) as tar:
112 for p in tar.getnames():
115 name = p.replace(fname + "/", "", 1)
116 if match is not None and not name.startswith(match):
118 name = name[len(match) + 1:]
120 fp = os.path.join(dst, name)
122 print(" extracting " + p)
124 tp = os.path.join(dst, p)
125 if os.path.isdir(tp) and os.path.exists(fp):
128 shutil.rmtree(os.path.join(dst, fname))
130 def run(args, verbose=False, exception=False, cwd=None):
132 print("running: " + ' '.join(args))
134 # Use Popen here instead of call() as it apparently allows powershell on
135 # Windows to not lock up waiting for input presumably.
136 ret = subprocess.Popen(args, cwd=cwd)
139 err = "failed to run: " + ' '.join(args)
140 if verbose or exception:
141 raise RuntimeError(err)
144 def stage0_data(rust_root):
145 nightlies = os.path.join(rust_root, "src/stage0.txt")
147 with open(nightlies, 'r') as nightlies:
148 for line in nightlies:
149 line = line.rstrip() # Strip newline character, '\n'
150 if line.startswith("#") or line == '':
152 a, b = line.split(": ", 1)
156 def format_build_time(duration):
157 return str(datetime.timedelta(seconds=int(duration)))
160 class RustBuild(object):
161 def download_stage0(self):
162 cache_dst = os.path.join(self.build_dir, "cache")
163 rustc_cache = os.path.join(cache_dst, self.stage0_date())
164 if not os.path.exists(rustc_cache):
165 os.makedirs(rustc_cache)
167 rustc_channel = self.stage0_rustc_channel()
168 cargo_channel = self.stage0_cargo_channel()
170 if self.rustc().startswith(self.bin_root()) and \
171 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
172 self.print_what_it_means_to_bootstrap()
173 if os.path.exists(self.bin_root()):
174 shutil.rmtree(self.bin_root())
175 filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
176 url = self._download_url + "/dist/" + self.stage0_date()
177 tarball = os.path.join(rustc_cache, filename)
178 if not os.path.exists(tarball):
179 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
180 unpack(tarball, self.bin_root(),
181 match="rust-std-" + self.build,
182 verbose=self.verbose)
184 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
185 url = self._download_url + "/dist/" + self.stage0_date()
186 tarball = os.path.join(rustc_cache, filename)
187 if not os.path.exists(tarball):
188 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
189 unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
190 self.fix_executable(self.bin_root() + "/bin/rustc")
191 self.fix_executable(self.bin_root() + "/bin/rustdoc")
192 with open(self.rustc_stamp(), 'w') as f:
193 f.write(self.stage0_date())
195 if "pc-windows-gnu" in self.build:
196 filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
197 url = self._download_url + "/dist/" + self.stage0_date()
198 tarball = os.path.join(rustc_cache, filename)
199 if not os.path.exists(tarball):
200 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
201 unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
203 if self.cargo().startswith(self.bin_root()) and \
204 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
205 self.print_what_it_means_to_bootstrap()
206 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
207 url = self._download_url + "/dist/" + self.stage0_date()
208 tarball = os.path.join(rustc_cache, filename)
209 if not os.path.exists(tarball):
210 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
211 unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
212 self.fix_executable(self.bin_root() + "/bin/cargo")
213 with open(self.cargo_stamp(), 'w') as f:
214 f.write(self.stage0_date())
216 def fix_executable(self, fname):
217 # If we're on NixOS we need to change the path to the dynamic loader
219 default_encoding = sys.getdefaultencoding()
221 ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
222 except (subprocess.CalledProcessError, WindowsError):
225 if ostype != "Linux":
228 if not os.path.exists("/etc/NIXOS"):
230 if os.path.exists("/lib"):
233 # At this point we're pretty sure the user is running NixOS
234 print("info: you seem to be running NixOS. Attempting to patch " + fname)
237 interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
238 interpreter = interpreter.strip().decode(default_encoding)
239 except subprocess.CalledProcessError as e:
240 print("warning: failed to call patchelf: %s" % e)
243 loader = interpreter.split("/")[-1]
246 ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
247 ldd_output = ldd_output.strip().decode(default_encoding)
248 except subprocess.CalledProcessError as e:
249 print("warning: unable to call ldd: %s" % e)
252 for line in ldd_output.splitlines():
253 libname = line.split()[0]
254 if libname.endswith(loader):
255 loader_path = libname[:len(libname) - len(loader)]
258 print("warning: unable to find the path to the dynamic linker")
261 correct_interpreter = loader_path + loader
264 subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
265 except subprocess.CalledProcessError as e:
266 print("warning: failed to call patchelf: %s" % e)
269 def stage0_date(self):
272 def stage0_rustc_channel(self):
273 return self._rustc_channel
275 def stage0_cargo_channel(self):
276 return self._cargo_channel
278 def rustc_stamp(self):
279 return os.path.join(self.bin_root(), '.rustc-stamp')
281 def cargo_stamp(self):
282 return os.path.join(self.bin_root(), '.cargo-stamp')
284 def rustc_out_of_date(self):
285 if not os.path.exists(self.rustc_stamp()) or self.clean:
287 with open(self.rustc_stamp(), 'r') as f:
288 return self.stage0_date() != f.read()
290 def cargo_out_of_date(self):
291 if not os.path.exists(self.cargo_stamp()) or self.clean:
293 with open(self.cargo_stamp(), 'r') as f:
294 return self.stage0_date() != f.read()
297 return os.path.join(self.build_dir, self.build, "stage0")
299 def get_toml(self, key):
300 for line in self.config_toml.splitlines():
301 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
302 if match is not None:
303 value = match.group(1)
304 return self.get_string(value) or value.strip()
307 def get_mk(self, key):
308 for line in iter(self.config_mk.splitlines()):
309 if line.startswith(key + ' '):
310 var = line[line.find(':=') + 2:].strip()
316 config = self.get_toml('cargo')
319 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
321 return config + '/bin/cargo' + self.exe_suffix()
322 return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
325 config = self.get_toml('rustc')
328 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
330 return config + '/bin/rustc' + self.exe_suffix()
331 return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
333 def get_string(self, line):
334 start = line.find('"')
337 end = start + 1 + line[start + 1:].find('"')
338 return line[start + 1:end]
340 def exe_suffix(self):
341 if sys.platform == 'win32':
346 def print_what_it_means_to_bootstrap(self):
347 if hasattr(self, 'printed'):
350 if os.path.exists(self.bootstrap_binary()):
352 if not '--help' in sys.argv or len(sys.argv) == 1:
355 print('info: the build system for Rust is written in Rust, so this')
356 print(' script is now going to download a stage0 rust compiler')
357 print(' and then compile the build system itself')
359 print('info: in the meantime you can read more about rustbuild at')
360 print(' src/bootstrap/README.md before the download finishes')
362 def bootstrap_binary(self):
363 return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
365 def build_bootstrap(self):
366 self.print_what_it_means_to_bootstrap()
367 build_dir = os.path.join(self.build_dir, "bootstrap")
368 if self.clean and os.path.exists(build_dir):
369 shutil.rmtree(build_dir)
370 env = os.environ.copy()
371 env["CARGO_TARGET_DIR"] = build_dir
372 env["RUSTC"] = self.rustc()
373 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
374 (os.pathsep + env["LD_LIBRARY_PATH"]) \
375 if "LD_LIBRARY_PATH" in env else ""
376 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
377 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
378 if "DYLD_LIBRARY_PATH" in env else ""
379 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
380 (os.pathsep + env["LIBRARY_PATH"]) \
381 if "LIBRARY_PATH" in env else ""
382 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
383 os.pathsep + env["PATH"]
384 if not os.path.isfile(self.cargo()):
385 raise Exception("no cargo executable found at `%s`" % self.cargo())
386 args = [self.cargo(), "build", "--manifest-path",
387 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
388 if self.use_locked_deps:
389 args.append("--locked")
390 if self.use_vendored_sources:
391 args.append("--frozen")
394 def run(self, args, env=None, cwd=None):
395 proc = subprocess.Popen(args, env=env, cwd=cwd)
400 def output(self, args, env=None, cwd=None):
401 proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
402 (out, err) = proc.communicate()
409 def build_triple(self):
410 default_encoding = sys.getdefaultencoding()
411 config = self.get_toml('build')
414 config = self.get_mk('CFG_BUILD')
418 ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
419 cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
420 except (subprocess.CalledProcessError, OSError):
421 if sys.platform == 'win32':
422 return 'x86_64-pc-windows-msvc'
423 err = "uname not found"
428 # The goal here is to come up with the same triple as LLVM would,
429 # at least for the subset of platforms we're willing to target.
430 if ostype == 'Linux':
431 os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
432 if os_from_sp == 'Android':
433 ostype = 'linux-android'
435 ostype = 'unknown-linux-gnu'
436 elif ostype == 'FreeBSD':
437 ostype = 'unknown-freebsd'
438 elif ostype == 'DragonFly':
439 ostype = 'unknown-dragonfly'
440 elif ostype == 'Bitrig':
441 ostype = 'unknown-bitrig'
442 elif ostype == 'OpenBSD':
443 ostype = 'unknown-openbsd'
444 elif ostype == 'NetBSD':
445 ostype = 'unknown-netbsd'
446 elif ostype == 'SunOS':
447 ostype = 'sun-solaris'
448 # On Solaris, uname -m will return a machine classification instead
449 # of a cpu type, so uname -p is recommended instead. However, the
450 # output from that option is too generic for our purposes (it will
451 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
452 # must be used instead.
454 cputype = subprocess.check_output(['isainfo',
455 '-k']).strip().decode(default_encoding)
456 except (subprocess.CalledProcessError, OSError):
457 err = "isainfo not found"
461 elif ostype == 'Darwin':
462 ostype = 'apple-darwin'
463 elif ostype == 'Haiku':
464 ostype = 'unknown-haiku'
465 elif ostype.startswith('MINGW'):
466 # msys' `uname` does not print gcc configuration, but prints msys
467 # configuration. so we cannot believe `uname -m`:
468 # msys1 is always i686 and msys2 is always x86_64.
469 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
471 ostype = 'pc-windows-gnu'
473 if os.environ.get('MSYSTEM') == 'MINGW64':
475 elif ostype.startswith('MSYS'):
476 ostype = 'pc-windows-gnu'
477 elif ostype.startswith('CYGWIN_NT'):
479 if ostype.endswith('WOW64'):
481 ostype = 'pc-windows-gnu'
483 err = "unknown OS type: " + ostype
485 raise ValueError(err)
488 if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
490 elif cputype in {'xscale', 'arm'}:
492 if ostype == 'linux-android':
493 ostype = 'linux-androideabi'
494 elif cputype == 'armv6l':
496 if ostype == 'linux-android':
497 ostype = 'linux-androideabi'
500 elif cputype in {'armv7l', 'armv8l'}:
502 if ostype == 'linux-android':
503 ostype = 'linux-androideabi'
506 elif cputype in {'aarch64', 'arm64'}:
508 elif cputype == 'mips':
509 if sys.byteorder == 'big':
511 elif sys.byteorder == 'little':
514 raise ValueError('unknown byteorder: ' + sys.byteorder)
515 elif cputype == 'mips64':
516 if sys.byteorder == 'big':
518 elif sys.byteorder == 'little':
521 raise ValueError('unknown byteorder: ' + sys.byteorder)
522 # only the n64 ABI is supported, indicate it
524 elif cputype in {'powerpc', 'ppc'}:
526 elif cputype in {'powerpc64', 'ppc64'}:
527 cputype = 'powerpc64'
528 elif cputype in {'powerpc64le', 'ppc64le'}:
529 cputype = 'powerpc64le'
530 elif cputype == 'sparcv9':
532 elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
534 elif cputype == 's390x':
536 elif cputype == 'BePC':
539 err = "unknown cpu type: " + cputype
541 raise ValueError(err)
544 return "{}-{}".format(cputype, ostype)
546 def update_submodules(self):
547 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
548 self.get_toml('submodules') == "false" or \
549 self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
552 print('Updating submodules')
553 output = self.output(["git", "submodule", "status"], cwd=self.rust_root)
555 for line in output.splitlines():
556 # NOTE `git submodule status` output looks like this:
558 # -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
559 # +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
560 # e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
562 # The first character can be '-', '+' or ' ' and denotes the
563 # `State` of the submodule Right next to this character is the
564 # SHA-1 of the submodule HEAD And after that comes the path to the
566 path = line[1:].split(' ')[1]
567 submodules.append([path, line[0]])
569 self.run(["git", "submodule", "sync"], cwd=self.rust_root)
571 for submod in submodules:
572 path, status = submod
573 if path.endswith(b"llvm") and \
574 (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
576 if path.endswith(b"jemalloc") and \
577 (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
579 submod_path = os.path.join(self.rust_root, path)
582 self.run(["git", "reset", "--hard"], cwd=submod_path)
583 self.run(["git", "clean", "-fdx"], cwd=submod_path)
585 self.run(["git", "submodule", "update", path], cwd=self.rust_root)
586 self.run(["git", "reset", "--hard"], cwd=submod_path)
587 self.run(["git", "clean", "-fdx"], cwd=submod_path)
589 self.run(["git", "submodule", "init", path], cwd=self.rust_root)
590 self.run(["git", "submodule", "update", path], cwd=self.rust_root)
592 raise ValueError('unknown submodule status: ' + status)
595 parser = argparse.ArgumentParser(description='Build rust')
596 parser.add_argument('--config')
597 parser.add_argument('--clean', action='store_true')
598 parser.add_argument('-v', '--verbose', action='store_true')
600 args = [a for a in sys.argv if a != '-h' and a != '--help']
601 args, _ = parser.parse_known_args(args)
603 # Configure initial bootstrap
607 rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
608 rb.build_dir = os.path.join(os.getcwd(), "build")
609 rb.verbose = args.verbose
610 rb.clean = args.clean
613 with open(args.config or 'config.toml') as config:
614 rb.config_toml = config.read()
618 rb.config_mk = open('config.mk').read()
622 rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
623 'CFG_ENABLE_VENDOR' in rb.config_mk
625 rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
626 'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
628 if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
629 if os.environ.get('USER') != os.environ['SUDO_USER']:
630 rb.use_vendored_sources = True
631 print('info: looks like you are running this command under `sudo`')
632 print(' and so in order to preserve your $HOME this will now')
633 print(' use vendored sources by default. Note that if this')
634 print(' does not work you should run a normal build first')
635 print(' before running a command like `sudo make install`')
637 if rb.use_vendored_sources:
638 if not os.path.exists('.cargo'):
639 os.makedirs('.cargo')
640 with open('.cargo/config','w') as f:
643 replace-with = 'vendored-sources'
644 registry = 'https://example.com'
646 [source.vendored-sources]
647 directory = '{}/src/vendor'
648 """.format(rb.rust_root))
650 if os.path.exists('.cargo'):
651 shutil.rmtree('.cargo')
653 data = stage0_data(rb.rust_root)
654 rb._date = data['date']
655 rb._rustc_channel = data['rustc']
656 rb._cargo_channel = data['cargo']
658 rb._download_url = 'https://dev-static.rust-lang.org'
660 rb._download_url = 'https://static.rust-lang.org'
662 rb.update_submodules()
664 # Fetch/build the bootstrap
665 rb.build = rb.build_triple()
672 args = [rb.bootstrap_binary()]
673 args.extend(sys.argv[1:])
674 env = os.environ.copy()
675 env["BUILD"] = rb.build
676 env["SRC"] = rb.rust_root
677 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
682 help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
685 if not help_triggered:
686 print("Build completed successfully in %s" % format_build_time(time() - start_time))
687 except (SystemExit, KeyboardInterrupt) as e:
688 if hasattr(e, 'code') and isinstance(e.code, int):
693 if not help_triggered:
694 print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
697 if __name__ == '__main__':