1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__ import print_function
27 def get(url, path, verbose=False):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
36 download(sha_path, sha_url, False, verbose)
37 if os.path.exists(path):
38 if verify(path, sha_path, False):
40 print("using already-download file", path)
44 print("ignoring already-download file",
45 path, "due to failed verification")
47 download(temp_path, url, True, verbose)
48 if not verify(temp_path, sha_path, verbose):
49 raise RuntimeError("failed verification")
51 print("moving {} to {}".format(temp_path, path))
52 shutil.move(temp_path, path)
54 delete_if_present(sha_path, verbose)
55 delete_if_present(temp_path, verbose)
58 def delete_if_present(path, verbose):
59 """Remove the given file if present"""
60 if os.path.isfile(path):
62 print("removing", path)
66 def download(path, url, probably_big, verbose):
69 _download(path, url, probably_big, verbose, True)
72 print("\nspurious failure, trying again")
73 _download(path, url, probably_big, verbose, False)
76 def _download(path, url, probably_big, verbose, exception):
77 if probably_big or verbose:
78 print("downloading {}".format(url))
79 # see http://serverfault.com/questions/301128/how-to-download
80 if sys.platform == 'win32':
81 run(["PowerShell.exe", "/nologo", "-Command",
82 "(New-Object System.Net.WebClient)"
83 ".DownloadFile('{}', '{}')".format(url, path)],
87 if probably_big or verbose:
91 run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
96 def verify(path, sha_path, verbose):
97 """Check if the sha256 sum of the given path is valid"""
99 print("verifying", path)
100 with open(path, "rb") as source:
101 found = hashlib.sha256(source.read()).hexdigest()
102 with open(sha_path, "r") as sha256sum:
103 expected = sha256sum.readline().split()[0]
104 verified = found == expected
106 print("invalid checksum:\n"
108 " expected: {}".format(found, expected))
112 def unpack(tarball, dst, verbose=False, match=None):
113 """Unpack the given tarball file"""
114 print("extracting", tarball)
115 fname = os.path.basename(tarball).replace(".tar.gz", "")
116 with contextlib.closing(tarfile.open(tarball)) as tar:
117 for member in tar.getnames():
118 if "/" not in member:
120 name = member.replace(fname + "/", "", 1)
121 if match is not None and not name.startswith(match):
123 name = name[len(match) + 1:]
125 dst_path = os.path.join(dst, name)
127 print(" extracting", member)
128 tar.extract(member, dst)
129 src_path = os.path.join(dst, member)
130 if os.path.isdir(src_path) and os.path.exists(dst_path):
132 shutil.move(src_path, dst_path)
133 shutil.rmtree(os.path.join(dst, fname))
136 def run(args, verbose=False, exception=False, **kwargs):
137 """Run a child program in a new process"""
139 print("running: " + ' '.join(args))
141 # Use Popen here instead of call() as it apparently allows powershell on
142 # Windows to not lock up waiting for input presumably.
143 ret = subprocess.Popen(args, **kwargs)
146 err = "failed to run: " + ' '.join(args)
147 if verbose or exception:
148 raise RuntimeError(err)
152 def stage0_data(rust_root):
153 """Build a dictionary from stage0.txt"""
154 nightlies = os.path.join(rust_root, "src/stage0.txt")
155 with open(nightlies, 'r') as nightlies:
156 lines = [line.rstrip() for line in nightlies
157 if not line.startswith("#")]
158 return dict([line.split(": ", 1) for line in lines if line])
161 def format_build_time(duration):
162 """Return a nicer format for build time
164 >>> format_build_time('300')
167 return str(datetime.timedelta(seconds=int(duration)))
170 class RustBuild(object):
171 """Provide all the methods required to build Rust"""
173 self.cargo_channel = ''
175 self._download_url = 'https://static.rust-lang.org'
176 self.rustc_channel = ''
178 self.build_dir = os.path.join(os.getcwd(), "build")
181 self.config_toml = ''
183 self.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
184 self.use_locked_deps = ''
185 self.use_vendored_sources = ''
188 def download_stage0(self):
189 """Fetch the build system for Rust, written in Rust
191 This method will build a cache directory, then it will fetch the
192 tarball which has the stage0 compiler used to then bootstrap the Rust
195 Each downloaded tarball is extracted, after that, the script
196 will move all the content to the right place.
198 rustc_channel = self.rustc_channel
199 cargo_channel = self.cargo_channel
201 if self.rustc().startswith(self.bin_root()) and \
202 (not os.path.exists(self.rustc()) or
203 self.program_out_of_date(self.rustc_stamp())):
204 self.print_what_bootstrap_means()
205 if os.path.exists(self.bin_root()):
206 shutil.rmtree(self.bin_root())
207 filename = "rust-std-{}-{}.tar.gz".format(
208 rustc_channel, self.build)
209 pattern = "rust-std-{}".format(self.build)
210 self._download_stage0_helper(filename, pattern)
212 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
213 self._download_stage0_helper(filename, "rustc")
214 self.fix_executable("{}/bin/rustc".format(self.bin_root()))
215 self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
216 with open(self.rustc_stamp(), 'w') as rust_stamp:
217 rust_stamp.write(self.date)
219 if "pc-windows-gnu" in self.build:
220 filename = "rust-mingw-{}-{}.tar.gz".format(
221 rustc_channel, self.build)
222 self._download_stage0_helper(filename, "rust-mingw")
224 if self.cargo().startswith(self.bin_root()) and \
225 (not os.path.exists(self.cargo()) or
226 self.program_out_of_date(self.cargo_stamp())):
227 self.print_what_bootstrap_means()
228 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
229 self._download_stage0_helper(filename, "cargo")
230 self.fix_executable("{}/bin/cargo".format(self.bin_root()))
231 with open(self.cargo_stamp(), 'w') as cargo_stamp:
232 cargo_stamp.write(self.date)
234 def _download_stage0_helper(self, filename, pattern):
235 cache_dst = os.path.join(self.build_dir, "cache")
236 rustc_cache = os.path.join(cache_dst, self.date)
237 if not os.path.exists(rustc_cache):
238 os.makedirs(rustc_cache)
240 url = "{}/dist/{}".format(self._download_url, self.date)
241 tarball = os.path.join(rustc_cache, filename)
242 if not os.path.exists(tarball):
243 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
244 unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
247 def fix_executable(fname):
248 """Modifies the interpreter section of 'fname' to fix the dynamic linker
250 This method is only required on NixOS and uses the PatchELF utility to
251 change the dynamic linker of ELF executables.
253 Please see https://nixos.org/patchelf.html for more information
255 default_encoding = sys.getdefaultencoding()
257 ostype = subprocess.check_output(
258 ['uname', '-s']).strip().decode(default_encoding)
259 except subprocess.CalledProcessError:
261 except OSError as reason:
262 if getattr(reason, 'winerror', None) is not None:
266 if ostype != "Linux":
269 if not os.path.exists("/etc/NIXOS"):
271 if os.path.exists("/lib"):
274 # At this point we're pretty sure the user is running NixOS
275 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
276 print(nix_os_msg, fname)
279 interpreter = subprocess.check_output(
280 ["patchelf", "--print-interpreter", fname])
281 interpreter = interpreter.strip().decode(default_encoding)
282 except subprocess.CalledProcessError as reason:
283 print("warning: failed to call patchelf:", reason)
286 loader = interpreter.split("/")[-1]
289 ldd_output = subprocess.check_output(
290 ['ldd', '/run/current-system/sw/bin/sh'])
291 ldd_output = ldd_output.strip().decode(default_encoding)
292 except subprocess.CalledProcessError as reason:
293 print("warning: unable to call ldd:", reason)
296 for line in ldd_output.splitlines():
297 libname = line.split()[0]
298 if libname.endswith(loader):
299 loader_path = libname[:len(libname) - len(loader)]
302 print("warning: unable to find the path to the dynamic linker")
305 correct_interpreter = loader_path + loader
308 subprocess.check_output(
309 ["patchelf", "--set-interpreter", correct_interpreter, fname])
310 except subprocess.CalledProcessError as reason:
311 print("warning: failed to call patchelf:", reason)
314 def rustc_stamp(self):
315 """Return the path for .rustc-stamp
318 >>> rb.build_dir = "build"
319 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
322 return os.path.join(self.bin_root(), '.rustc-stamp')
324 def cargo_stamp(self):
325 """Return the path for .cargo-stamp
328 >>> rb.build_dir = "build"
329 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
332 return os.path.join(self.bin_root(), '.cargo-stamp')
334 def program_out_of_date(self, stamp_path):
335 """Check if the given program stamp is out of date"""
336 if not os.path.exists(stamp_path) or self.clean:
338 with open(stamp_path, 'r') as stamp:
339 return self.date != stamp.read()
342 """Return the binary root directory
345 >>> rb.build_dir = "build"
346 >>> rb.bin_root() == os.path.join("build", "stage0")
349 When the 'build' property is given should be a nested directory:
351 >>> rb.build = "devel"
352 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
355 return os.path.join(self.build_dir, self.build, "stage0")
357 def get_toml(self, key):
358 """Returns the value of the given key in config.toml, otherwise returns None
361 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
362 >>> rb.get_toml("key2")
365 If the key does not exists, the result is None:
367 >>> rb.get_toml("key3") == None
370 for line in self.config_toml.splitlines():
371 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
372 if match is not None:
373 value = match.group(1)
374 return self.get_string(value) or value.strip()
377 def get_mk(self, key):
378 """Returns the value of the given key in config.mk, otherwise returns None
381 >>> rb.config_mk = 'key := value\\n'
385 If the key does not exists, the result is None:
387 >>> rb.get_mk('does_not_exists') == None
390 for line in iter(self.config_mk.splitlines()):
391 if line.startswith(key + ' '):
392 var = line[line.find(':=') + 2:].strip()
398 """Return config path for cargo"""
399 return self.program_config('cargo')
402 """Return config path for rustc"""
403 return self.program_config('rustc')
405 def program_config(self, program):
406 """Return config path for the given program
409 >>> rb.config_toml = 'rustc = "rustc"\\n'
410 >>> rb.config_mk = 'CFG_LOCAL_RUST_ROOT := /tmp/rust\\n'
411 >>> rb.program_config('rustc')
413 >>> cargo_path = rb.program_config('cargo')
414 >>> cargo_path.rstrip(".exe") == os.path.join("/tmp/rust",
417 >>> rb.config_toml = ''
418 >>> rb.config_mk = ''
419 >>> cargo_path = rb.program_config('cargo')
420 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
424 config = self.get_toml(program)
427 config = self.get_mk('CFG_LOCAL_RUST_ROOT')
429 return os.path.join(config, "bin", "{}{}".format(
430 program, self.exe_suffix()))
431 return os.path.join(self.bin_root(), "bin", "{}{}".format(
432 program, self.exe_suffix()))
435 def get_string(line):
436 """Return the value between double quotes
438 >>> RustBuild.get_string(' "devel" ')
441 start = line.find('"')
444 end = start + 1 + line[start + 1:].find('"')
445 return line[start + 1:end]
449 """Return a suffix for executables"""
450 if sys.platform == 'win32':
454 def print_what_bootstrap_means(self):
455 """Prints more information about the build system"""
456 if hasattr(self, 'printed'):
459 if os.path.exists(self.bootstrap_binary()):
461 if '--help' not in sys.argv or len(sys.argv) == 1:
464 print('info: the build system for Rust is written in Rust, so this')
465 print(' script is now going to download a stage0 rust compiler')
466 print(' and then compile the build system itself')
468 print('info: in the meantime you can read more about rustbuild at')
469 print(' src/bootstrap/README.md before the download finishes')
471 def bootstrap_binary(self):
472 """Return the path of the boostrap binary
475 >>> rb.build_dir = "build"
476 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
477 ... "debug", "bootstrap")
480 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
482 def build_bootstrap(self):
483 """Build bootstrap"""
484 self.print_what_bootstrap_means()
485 build_dir = os.path.join(self.build_dir, "bootstrap")
486 if self.clean and os.path.exists(build_dir):
487 shutil.rmtree(build_dir)
488 env = os.environ.copy()
489 env["RUSTC_BOOTSTRAP"] = '1'
490 env["CARGO_TARGET_DIR"] = build_dir
491 env["RUSTC"] = self.rustc()
492 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
493 (os.pathsep + env["LD_LIBRARY_PATH"]) \
494 if "LD_LIBRARY_PATH" in env else ""
495 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
496 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
497 if "DYLD_LIBRARY_PATH" in env else ""
498 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
499 (os.pathsep + env["LIBRARY_PATH"]) \
500 if "LIBRARY_PATH" in env else ""
501 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
502 os.pathsep + env["PATH"]
503 if not os.path.isfile(self.cargo()):
504 raise Exception("no cargo executable found at `{}`".format(
506 args = [self.cargo(), "build", "--manifest-path",
507 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
509 args.append("--verbose")
511 args.append("--verbose")
512 if self.use_locked_deps:
513 args.append("--locked")
514 if self.use_vendored_sources:
515 args.append("--frozen")
516 run(args, env=env, verbose=self.verbose)
518 def build_triple(self):
519 """Build triple as in LLVM"""
520 default_encoding = sys.getdefaultencoding()
521 config = self.get_toml('build')
524 config = self.get_mk('CFG_BUILD')
528 ostype = subprocess.check_output(
529 ['uname', '-s']).strip().decode(default_encoding)
530 cputype = subprocess.check_output(
531 ['uname', '-m']).strip().decode(default_encoding)
532 except (subprocess.CalledProcessError, OSError):
533 if sys.platform == 'win32':
534 return 'x86_64-pc-windows-msvc'
535 err = "uname not found"
540 # The goal here is to come up with the same triple as LLVM would,
541 # at least for the subset of platforms we're willing to target.
543 'Bitrig': 'unknown-bitrig',
544 'Darwin': 'apple-darwin',
545 'DragonFly': 'unknown-dragonfly',
546 'FreeBSD': 'unknown-freebsd',
547 'Haiku': 'unknown-haiku',
548 'NetBSD': 'unknown-netbsd',
549 'OpenBSD': 'unknown-openbsd'
552 # Consider the direct transformation first and then the special cases
553 if ostype in ostype_mapper:
554 ostype = ostype_mapper[ostype]
555 elif ostype == 'Linux':
556 os_from_sp = subprocess.check_output(
557 ['uname', '-o']).strip().decode(default_encoding)
558 if os_from_sp == 'Android':
559 ostype = 'linux-android'
561 ostype = 'unknown-linux-gnu'
562 elif ostype == 'SunOS':
563 ostype = 'sun-solaris'
564 # On Solaris, uname -m will return a machine classification instead
565 # of a cpu type, so uname -p is recommended instead. However, the
566 # output from that option is too generic for our purposes (it will
567 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
568 # must be used instead.
570 cputype = subprocess.check_output(
571 ['isainfo', '-k']).strip().decode(default_encoding)
572 except (subprocess.CalledProcessError, OSError):
573 err = "isainfo not found"
577 elif ostype.startswith('MINGW'):
578 # msys' `uname` does not print gcc configuration, but prints msys
579 # configuration. so we cannot believe `uname -m`:
580 # msys1 is always i686 and msys2 is always x86_64.
581 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
583 ostype = 'pc-windows-gnu'
585 if os.environ.get('MSYSTEM') == 'MINGW64':
587 elif ostype.startswith('MSYS'):
588 ostype = 'pc-windows-gnu'
589 elif ostype.startswith('CYGWIN_NT'):
591 if ostype.endswith('WOW64'):
593 ostype = 'pc-windows-gnu'
595 err = "unknown OS type: {}".format(ostype)
597 raise ValueError(err)
602 'aarch64': 'aarch64',
609 'powerpc': 'powerpc',
610 'powerpc64': 'powerpc64',
611 'powerpc64le': 'powerpc64le',
613 'ppc64': 'powerpc64',
614 'ppc64le': 'powerpc64le',
622 # Consider the direct transformation first and then the special cases
623 if cputype in cputype_mapper:
624 cputype = cputype_mapper[cputype]
625 elif cputype in {'xscale', 'arm'}:
627 if ostype == 'linux-android':
628 ostype = 'linux-androideabi'
629 elif cputype == 'armv6l':
631 if ostype == 'linux-android':
632 ostype = 'linux-androideabi'
635 elif cputype in {'armv7l', 'armv8l'}:
637 if ostype == 'linux-android':
638 ostype = 'linux-androideabi'
641 elif cputype == 'mips':
642 if sys.byteorder == 'big':
644 elif sys.byteorder == 'little':
647 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
648 elif cputype == 'mips64':
649 if sys.byteorder == 'big':
651 elif sys.byteorder == 'little':
654 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
655 # only the n64 ABI is supported, indicate it
657 elif cputype == 'sparcv9':
660 err = "unknown cpu type: {}".format(cputype)
662 raise ValueError(err)
665 return "{}-{}".format(cputype, ostype)
667 def update_submodules(self):
668 """Update submodules"""
669 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
670 self.get_toml('submodules') == "false" or \
671 self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
673 print('Updating submodules')
674 default_encoding = sys.getdefaultencoding()
675 run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
676 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
677 ["git", "config", "--file",
678 os.path.join(self.rust_root, ".gitmodules"),
679 "--get-regexp", "path"]
680 ).decode(default_encoding).splitlines()]
681 submodules = [module for module in submodules
682 if not ((module.endswith("llvm") and
683 (self.get_toml('llvm-config') or
684 self.get_mk('CFG_LLVM_ROOT'))) or
685 (module.endswith("jemalloc") and
686 (self.get_toml('jemalloc') or
687 self.get_mk('CFG_JEMALLOC_ROOT'))))]
688 run(["git", "submodule", "update",
689 "--init", "--recursive"] + submodules,
690 cwd=self.rust_root, verbose=self.verbose)
691 run(["git", "submodule", "-q", "foreach", "git",
692 "reset", "-q", "--hard"],
693 cwd=self.rust_root, verbose=self.verbose)
694 run(["git", "submodule", "-q", "foreach", "git",
696 cwd=self.rust_root, verbose=self.verbose)
698 def set_dev_environment(self):
699 """Set download URL for development environment"""
700 self._download_url = 'https://dev-static.rust-lang.org'
704 """Configure, fetch, build and run the initial bootstrap"""
705 parser = argparse.ArgumentParser(description='Build rust')
706 parser.add_argument('--config')
707 parser.add_argument('--build')
708 parser.add_argument('--clean', action='store_true')
709 parser.add_argument('-v', '--verbose', action='store_true')
711 args = [a for a in sys.argv if a != '-h' and a != '--help']
712 args, _ = parser.parse_known_args(args)
714 # Configure initial bootstrap
716 build.verbose = args.verbose
717 build.clean = args.clean
720 with open(args.config or 'config.toml') as config:
721 build.config_toml = config.read()
725 build.config_mk = open('config.mk').read()
729 if '\nverbose = 2' in build.config_toml:
731 elif '\nverbose = 1' in build.config_toml:
734 build.use_vendored_sources = '\nvendor = true' in build.config_toml or \
735 'CFG_ENABLE_VENDOR' in build.config_mk
737 build.use_locked_deps = '\nlocked-deps = true' in build.config_toml or \
738 'CFG_ENABLE_LOCKED_DEPS' in build.config_mk
740 if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
741 if os.environ.get('USER') != os.environ['SUDO_USER']:
742 build.use_vendored_sources = True
743 print('info: looks like you are running this command under `sudo`')
744 print(' and so in order to preserve your $HOME this will now')
745 print(' use vendored sources by default. Note that if this')
746 print(' does not work you should run a normal build first')
747 print(' before running a command like `sudo make install`')
749 if build.use_vendored_sources:
750 if not os.path.exists('.cargo'):
751 os.makedirs('.cargo')
752 with open('.cargo/config', 'w') as cargo_config:
753 cargo_config.write("""
755 replace-with = 'vendored-sources'
756 registry = 'https://example.com'
758 [source.vendored-sources]
759 directory = '{}/src/vendor'
760 """.format(build.rust_root))
762 if os.path.exists('.cargo'):
763 shutil.rmtree('.cargo')
765 data = stage0_data(build.rust_root)
766 build.date = data['date']
767 build.rustc_channel = data['rustc']
768 build.cargo_channel = data['cargo']
771 build.set_dev_environment()
773 build.update_submodules()
775 # Fetch/build the bootstrap
776 build.build = args.build or build.build_triple()
777 build.download_stage0()
779 build.build_bootstrap()
783 args = [build.bootstrap_binary()]
784 args.extend(sys.argv[1:])
785 env = os.environ.copy()
786 env["BUILD"] = build.build
787 env["SRC"] = build.rust_root
788 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
789 env["BOOTSTRAP_PYTHON"] = sys.executable
790 run(args, env=env, verbose=build.verbose)
794 """Entry point for the bootstrap process"""
797 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
800 if not help_triggered:
801 print("Build completed successfully in {}".format(
802 format_build_time(time() - start_time)))
803 except (SystemExit, KeyboardInterrupt) as error:
804 if hasattr(error, 'code') and isinstance(error.code, int):
805 exit_code = error.code
809 if not help_triggered:
810 print("Build completed unsuccessfully in {}".format(
811 format_build_time(time() - start_time)))
815 if __name__ == '__main__':