1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__ import print_function
27 def get(url, path, verbose=False):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
36 download(sha_path, sha_url, False, verbose)
37 if os.path.exists(path):
38 if verify(path, sha_path, False):
40 print("using already-download file", path)
44 print("ignoring already-download file",
45 path, "due to failed verification")
47 download(temp_path, url, True, verbose)
48 if not verify(temp_path, sha_path, verbose):
49 raise RuntimeError("failed verification")
51 print("moving {} to {}".format(temp_path, path))
52 shutil.move(temp_path, path)
54 delete_if_present(sha_path, verbose)
55 delete_if_present(temp_path, verbose)
58 def delete_if_present(path, verbose):
59 """Remove the given file if present"""
60 if os.path.isfile(path):
62 print("removing", path)
66 def download(path, url, probably_big, verbose):
69 _download(path, url, probably_big, verbose, True)
72 print("\nspurious failure, trying again")
73 _download(path, url, probably_big, verbose, False)
76 def _download(path, url, probably_big, verbose, exception):
77 if probably_big or verbose:
78 print("downloading {}".format(url))
79 # see http://serverfault.com/questions/301128/how-to-download
80 if sys.platform == 'win32':
81 run(["PowerShell.exe", "/nologo", "-Command",
82 "(New-Object System.Net.WebClient)"
83 ".DownloadFile('{}', '{}')".format(url, path)],
87 if probably_big or verbose:
91 run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
96 def verify(path, sha_path, verbose):
97 """Check if the sha256 sum of the given path is valid"""
99 print("verifying", path)
100 with open(path, "rb") as source:
101 found = hashlib.sha256(source.read()).hexdigest()
102 with open(sha_path, "r") as sha256sum:
103 expected = sha256sum.readline().split()[0]
104 verified = found == expected
106 print("invalid checksum:\n"
108 " expected: {}".format(found, expected))
112 def unpack(tarball, dst, verbose=False, match=None):
113 """Unpack the given tarball file"""
114 print("extracting", tarball)
115 fname = os.path.basename(tarball).replace(".tar.gz", "")
116 with contextlib.closing(tarfile.open(tarball)) as tar:
117 for member in tar.getnames():
118 if "/" not in member:
120 name = member.replace(fname + "/", "", 1)
121 if match is not None and not name.startswith(match):
123 name = name[len(match) + 1:]
125 dst_path = os.path.join(dst, name)
127 print(" extracting", member)
128 tar.extract(member, dst)
129 src_path = os.path.join(dst, member)
130 if os.path.isdir(src_path) and os.path.exists(dst_path):
132 shutil.move(src_path, dst_path)
133 shutil.rmtree(os.path.join(dst, fname))
136 def run(args, verbose=False, exception=False, **kwargs):
137 """Run a child program in a new process"""
139 print("running: " + ' '.join(args))
141 # Use Popen here instead of call() as it apparently allows powershell on
142 # Windows to not lock up waiting for input presumably.
143 ret = subprocess.Popen(args, **kwargs)
146 err = "failed to run: " + ' '.join(args)
147 if verbose or exception:
148 raise RuntimeError(err)
152 def stage0_data(rust_root):
153 """Build a dictionary from stage0.txt"""
154 nightlies = os.path.join(rust_root, "src/stage0.txt")
155 with open(nightlies, 'r') as nightlies:
156 lines = [line.rstrip() for line in nightlies
157 if not line.startswith("#")]
158 return dict([line.split(": ", 1) for line in lines if line])
161 def format_build_time(duration):
162 """Return a nicer format for build time
164 >>> format_build_time('300')
167 return str(datetime.timedelta(seconds=int(duration)))
170 def default_build_triple():
171 """Build triple as in LLVM"""
172 default_encoding = sys.getdefaultencoding()
174 ostype = subprocess.check_output(
175 ['uname', '-s']).strip().decode(default_encoding)
176 cputype = subprocess.check_output(
177 ['uname', '-m']).strip().decode(default_encoding)
178 except (subprocess.CalledProcessError, OSError):
179 if sys.platform == 'win32':
180 return 'x86_64-pc-windows-msvc'
181 err = "uname not found"
184 # The goal here is to come up with the same triple as LLVM would,
185 # at least for the subset of platforms we're willing to target.
187 'Bitrig': 'unknown-bitrig',
188 'Darwin': 'apple-darwin',
189 'DragonFly': 'unknown-dragonfly',
190 'FreeBSD': 'unknown-freebsd',
191 'Haiku': 'unknown-haiku',
192 'NetBSD': 'unknown-netbsd',
193 'OpenBSD': 'unknown-openbsd'
196 # Consider the direct transformation first and then the special cases
197 if ostype in ostype_mapper:
198 ostype = ostype_mapper[ostype]
199 elif ostype == 'Linux':
200 os_from_sp = subprocess.check_output(
201 ['uname', '-o']).strip().decode(default_encoding)
202 if os_from_sp == 'Android':
203 ostype = 'linux-android'
205 ostype = 'unknown-linux-gnu'
206 elif ostype == 'SunOS':
207 ostype = 'sun-solaris'
208 # On Solaris, uname -m will return a machine classification instead
209 # of a cpu type, so uname -p is recommended instead. However, the
210 # output from that option is too generic for our purposes (it will
211 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
212 # must be used instead.
214 cputype = subprocess.check_output(
215 ['isainfo', '-k']).strip().decode(default_encoding)
216 except (subprocess.CalledProcessError, OSError):
217 err = "isainfo not found"
219 elif ostype.startswith('MINGW'):
220 # msys' `uname` does not print gcc configuration, but prints msys
221 # configuration. so we cannot believe `uname -m`:
222 # msys1 is always i686 and msys2 is always x86_64.
223 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
225 ostype = 'pc-windows-gnu'
227 if os.environ.get('MSYSTEM') == 'MINGW64':
229 elif ostype.startswith('MSYS'):
230 ostype = 'pc-windows-gnu'
231 elif ostype.startswith('CYGWIN_NT'):
233 if ostype.endswith('WOW64'):
235 ostype = 'pc-windows-gnu'
237 err = "unknown OS type: {}".format(ostype)
242 'aarch64': 'aarch64',
249 'powerpc': 'powerpc',
250 'powerpc64': 'powerpc64',
251 'powerpc64le': 'powerpc64le',
253 'ppc64': 'powerpc64',
254 'ppc64le': 'powerpc64le',
262 # Consider the direct transformation first and then the special cases
263 if cputype in cputype_mapper:
264 cputype = cputype_mapper[cputype]
265 elif cputype in {'xscale', 'arm'}:
267 if ostype == 'linux-android':
268 ostype = 'linux-androideabi'
269 elif cputype == 'armv6l':
271 if ostype == 'linux-android':
272 ostype = 'linux-androideabi'
275 elif cputype in {'armv7l', 'armv8l'}:
277 if ostype == 'linux-android':
278 ostype = 'linux-androideabi'
281 elif cputype == 'mips':
282 if sys.byteorder == 'big':
284 elif sys.byteorder == 'little':
287 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
288 elif cputype == 'mips64':
289 if sys.byteorder == 'big':
291 elif sys.byteorder == 'little':
294 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
295 # only the n64 ABI is supported, indicate it
297 elif cputype == 'sparcv9':
300 err = "unknown cpu type: {}".format(cputype)
303 return "{}-{}".format(cputype, ostype)
305 class RustBuild(object):
306 """Provide all the methods required to build Rust"""
308 self.cargo_channel = ''
310 self._download_url = 'https://static.rust-lang.org'
311 self.rustc_channel = ''
313 self.build_dir = os.path.join(os.getcwd(), "build")
315 self.config_toml = ''
317 self.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
318 self.use_locked_deps = ''
319 self.use_vendored_sources = ''
322 def download_stage0(self):
323 """Fetch the build system for Rust, written in Rust
325 This method will build a cache directory, then it will fetch the
326 tarball which has the stage0 compiler used to then bootstrap the Rust
329 Each downloaded tarball is extracted, after that, the script
330 will move all the content to the right place.
332 rustc_channel = self.rustc_channel
333 cargo_channel = self.cargo_channel
335 if self.rustc().startswith(self.bin_root()) and \
336 (not os.path.exists(self.rustc()) or
337 self.program_out_of_date(self.rustc_stamp())):
338 self.print_what_bootstrap_means()
339 if os.path.exists(self.bin_root()):
340 shutil.rmtree(self.bin_root())
341 filename = "rust-std-{}-{}.tar.gz".format(
342 rustc_channel, self.build)
343 pattern = "rust-std-{}".format(self.build)
344 self._download_stage0_helper(filename, pattern)
346 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
347 self._download_stage0_helper(filename, "rustc")
348 self.fix_executable("{}/bin/rustc".format(self.bin_root()))
349 self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
350 with open(self.rustc_stamp(), 'w') as rust_stamp:
351 rust_stamp.write(self.date)
353 if "pc-windows-gnu" in self.build:
354 filename = "rust-mingw-{}-{}.tar.gz".format(
355 rustc_channel, self.build)
356 self._download_stage0_helper(filename, "rust-mingw")
358 if self.cargo().startswith(self.bin_root()) and \
359 (not os.path.exists(self.cargo()) or
360 self.program_out_of_date(self.cargo_stamp())):
361 self.print_what_bootstrap_means()
362 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
363 self._download_stage0_helper(filename, "cargo")
364 self.fix_executable("{}/bin/cargo".format(self.bin_root()))
365 with open(self.cargo_stamp(), 'w') as cargo_stamp:
366 cargo_stamp.write(self.date)
368 def _download_stage0_helper(self, filename, pattern):
369 cache_dst = os.path.join(self.build_dir, "cache")
370 rustc_cache = os.path.join(cache_dst, self.date)
371 if not os.path.exists(rustc_cache):
372 os.makedirs(rustc_cache)
374 url = "{}/dist/{}".format(self._download_url, self.date)
375 tarball = os.path.join(rustc_cache, filename)
376 if not os.path.exists(tarball):
377 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
378 unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
381 def fix_executable(fname):
382 """Modifies the interpreter section of 'fname' to fix the dynamic linker
384 This method is only required on NixOS and uses the PatchELF utility to
385 change the dynamic linker of ELF executables.
387 Please see https://nixos.org/patchelf.html for more information
389 default_encoding = sys.getdefaultencoding()
391 ostype = subprocess.check_output(
392 ['uname', '-s']).strip().decode(default_encoding)
393 except subprocess.CalledProcessError:
395 except OSError as reason:
396 if getattr(reason, 'winerror', None) is not None:
400 if ostype != "Linux":
403 if not os.path.exists("/etc/NIXOS"):
405 if os.path.exists("/lib"):
408 # At this point we're pretty sure the user is running NixOS
409 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
410 print(nix_os_msg, fname)
413 interpreter = subprocess.check_output(
414 ["patchelf", "--print-interpreter", fname])
415 interpreter = interpreter.strip().decode(default_encoding)
416 except subprocess.CalledProcessError as reason:
417 print("warning: failed to call patchelf:", reason)
420 loader = interpreter.split("/")[-1]
423 ldd_output = subprocess.check_output(
424 ['ldd', '/run/current-system/sw/bin/sh'])
425 ldd_output = ldd_output.strip().decode(default_encoding)
426 except subprocess.CalledProcessError as reason:
427 print("warning: unable to call ldd:", reason)
430 for line in ldd_output.splitlines():
431 libname = line.split()[0]
432 if libname.endswith(loader):
433 loader_path = libname[:len(libname) - len(loader)]
436 print("warning: unable to find the path to the dynamic linker")
439 correct_interpreter = loader_path + loader
442 subprocess.check_output(
443 ["patchelf", "--set-interpreter", correct_interpreter, fname])
444 except subprocess.CalledProcessError as reason:
445 print("warning: failed to call patchelf:", reason)
448 def rustc_stamp(self):
449 """Return the path for .rustc-stamp
452 >>> rb.build_dir = "build"
453 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
456 return os.path.join(self.bin_root(), '.rustc-stamp')
458 def cargo_stamp(self):
459 """Return the path for .cargo-stamp
462 >>> rb.build_dir = "build"
463 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
466 return os.path.join(self.bin_root(), '.cargo-stamp')
468 def program_out_of_date(self, stamp_path):
469 """Check if the given program stamp is out of date"""
470 if not os.path.exists(stamp_path) or self.clean:
472 with open(stamp_path, 'r') as stamp:
473 return self.date != stamp.read()
476 """Return the binary root directory
479 >>> rb.build_dir = "build"
480 >>> rb.bin_root() == os.path.join("build", "stage0")
483 When the 'build' property is given should be a nested directory:
485 >>> rb.build = "devel"
486 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
489 return os.path.join(self.build_dir, self.build, "stage0")
491 def get_toml(self, key):
492 """Returns the value of the given key in config.toml, otherwise returns None
495 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
496 >>> rb.get_toml("key2")
499 If the key does not exists, the result is None:
501 >>> rb.get_toml("key3") == None
504 for line in self.config_toml.splitlines():
505 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
506 if match is not None:
507 value = match.group(1)
508 return self.get_string(value) or value.strip()
512 """Return config path for cargo"""
513 return self.program_config('cargo')
516 """Return config path for rustc"""
517 return self.program_config('rustc')
519 def program_config(self, program):
520 """Return config path for the given program
523 >>> rb.config_toml = 'rustc = "rustc"\\n'
524 >>> rb.program_config('rustc')
526 >>> cargo_path = rb.program_config('cargo')
527 >>> cargo_path.rstrip(".exe") == os.path.join("/tmp/rust",
530 >>> rb.config_toml = ''
531 >>> cargo_path = rb.program_config('cargo')
532 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
536 config = self.get_toml(program)
539 return os.path.join(self.bin_root(), "bin", "{}{}".format(
540 program, self.exe_suffix()))
543 def get_string(line):
544 """Return the value between double quotes
546 >>> RustBuild.get_string(' "devel" ')
549 start = line.find('"')
551 end = start + 1 + line[start + 1:].find('"')
552 return line[start + 1:end]
553 start = line.find('\'')
555 end = start + 1 + line[start + 1:].find('\'')
556 return line[start + 1:end]
561 """Return a suffix for executables"""
562 if sys.platform == 'win32':
566 def print_what_bootstrap_means(self):
567 """Prints more information about the build system"""
568 if hasattr(self, 'printed'):
571 if os.path.exists(self.bootstrap_binary()):
573 if '--help' not in sys.argv or len(sys.argv) == 1:
576 print('info: the build system for Rust is written in Rust, so this')
577 print(' script is now going to download a stage0 rust compiler')
578 print(' and then compile the build system itself')
580 print('info: in the meantime you can read more about rustbuild at')
581 print(' src/bootstrap/README.md before the download finishes')
583 def bootstrap_binary(self):
584 """Return the path of the boostrap binary
587 >>> rb.build_dir = "build"
588 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
589 ... "debug", "bootstrap")
592 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
594 def build_bootstrap(self):
595 """Build bootstrap"""
596 self.print_what_bootstrap_means()
597 build_dir = os.path.join(self.build_dir, "bootstrap")
598 if self.clean and os.path.exists(build_dir):
599 shutil.rmtree(build_dir)
600 env = os.environ.copy()
601 env["RUSTC_BOOTSTRAP"] = '1'
602 env["CARGO_TARGET_DIR"] = build_dir
603 env["RUSTC"] = self.rustc()
604 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
605 (os.pathsep + env["LD_LIBRARY_PATH"]) \
606 if "LD_LIBRARY_PATH" in env else ""
607 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
608 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
609 if "DYLD_LIBRARY_PATH" in env else ""
610 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
611 (os.pathsep + env["LIBRARY_PATH"]) \
612 if "LIBRARY_PATH" in env else ""
613 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
614 os.pathsep + env["PATH"]
615 if not os.path.isfile(self.cargo()):
616 raise Exception("no cargo executable found at `{}`".format(
618 args = [self.cargo(), "build", "--manifest-path",
619 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
621 args.append("--verbose")
623 args.append("--verbose")
624 if self.use_locked_deps:
625 args.append("--locked")
626 if self.use_vendored_sources:
627 args.append("--frozen")
628 run(args, env=env, verbose=self.verbose)
630 def build_triple(self):
631 """Build triple as in LLVM"""
632 default_encoding = sys.getdefaultencoding()
633 config = self.get_toml('build')
636 return default_build_triple()
638 def update_submodules(self):
639 """Update submodules"""
640 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
641 self.get_toml('submodules') == "false":
643 print('Updating submodules')
644 default_encoding = sys.getdefaultencoding()
645 run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
646 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
647 ["git", "config", "--file",
648 os.path.join(self.rust_root, ".gitmodules"),
649 "--get-regexp", "path"]
650 ).decode(default_encoding).splitlines()]
651 submodules = [module for module in submodules
652 if not ((module.endswith("llvm") and
653 self.get_toml('llvm-config')) or
654 (module.endswith("jemalloc") and
655 self.get_toml('jemalloc')))]
656 run(["git", "submodule", "update",
657 "--init", "--recursive"] + submodules,
658 cwd=self.rust_root, verbose=self.verbose)
659 run(["git", "submodule", "-q", "foreach", "git",
660 "reset", "-q", "--hard"],
661 cwd=self.rust_root, verbose=self.verbose)
662 run(["git", "submodule", "-q", "foreach", "git",
664 cwd=self.rust_root, verbose=self.verbose)
666 def set_dev_environment(self):
667 """Set download URL for development environment"""
668 self._download_url = 'https://dev-static.rust-lang.org'
672 """Configure, fetch, build and run the initial bootstrap"""
673 parser = argparse.ArgumentParser(description='Build rust')
674 parser.add_argument('--config')
675 parser.add_argument('--build')
676 parser.add_argument('--clean', action='store_true')
677 parser.add_argument('-v', '--verbose', action='store_true')
679 args = [a for a in sys.argv if a != '-h' and a != '--help']
680 args, _ = parser.parse_known_args(args)
682 # Configure initial bootstrap
684 build.verbose = args.verbose
685 build.clean = args.clean
688 with open(args.config or 'config.toml') as config:
689 build.config_toml = config.read()
693 if '\nverbose = 2' in build.config_toml:
695 elif '\nverbose = 1' in build.config_toml:
698 build.use_vendored_sources = '\nvendor = true' in build.config_toml
700 build.use_locked_deps = '\nlocked-deps = true' in build.config_toml
702 if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
703 if os.environ.get('USER') != os.environ['SUDO_USER']:
704 build.use_vendored_sources = True
705 print('info: looks like you are running this command under `sudo`')
706 print(' and so in order to preserve your $HOME this will now')
707 print(' use vendored sources by default. Note that if this')
708 print(' does not work you should run a normal build first')
709 print(' before running a command like `sudo make install`')
711 if build.use_vendored_sources:
712 if not os.path.exists('.cargo'):
713 os.makedirs('.cargo')
714 with open('.cargo/config', 'w') as cargo_config:
715 cargo_config.write("""
717 replace-with = 'vendored-sources'
718 registry = 'https://example.com'
720 [source.vendored-sources]
721 directory = '{}/src/vendor'
722 """.format(build.rust_root))
724 if os.path.exists('.cargo'):
725 shutil.rmtree('.cargo')
727 data = stage0_data(build.rust_root)
728 build.date = data['date']
729 build.rustc_channel = data['rustc']
730 build.cargo_channel = data['cargo']
733 build.set_dev_environment()
735 build.update_submodules()
737 # Fetch/build the bootstrap
738 build.build = args.build or build.build_triple()
739 build.download_stage0()
741 build.build_bootstrap()
745 args = [build.bootstrap_binary()]
746 args.extend(sys.argv[1:])
747 env = os.environ.copy()
748 env["BUILD"] = build.build
749 env["SRC"] = build.rust_root
750 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
751 env["BOOTSTRAP_PYTHON"] = sys.executable
752 run(args, env=env, verbose=build.verbose)
756 """Entry point for the bootstrap process"""
759 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
762 if not help_triggered:
763 print("Build completed successfully in {}".format(
764 format_build_time(time() - start_time)))
765 except (SystemExit, KeyboardInterrupt) as error:
766 if hasattr(error, 'code') and isinstance(error.code, int):
767 exit_code = error.code
771 if not help_triggered:
772 print("Build completed unsuccessfully in {}".format(
773 format_build_time(time() - start_time)))
777 if __name__ == '__main__':