1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__ import print_function
27 def get(url, path, verbose=False):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
36 download(sha_path, sha_url, False, verbose)
37 if os.path.exists(path):
38 if verify(path, sha_path, False):
40 print("using already-download file", path)
44 print("ignoring already-download file",
45 path, "due to failed verification")
47 download(temp_path, url, True, verbose)
48 if not verify(temp_path, sha_path, verbose):
49 raise RuntimeError("failed verification")
51 print("moving {} to {}".format(temp_path, path))
52 shutil.move(temp_path, path)
54 delete_if_present(sha_path, verbose)
55 delete_if_present(temp_path, verbose)
58 def delete_if_present(path, verbose):
59 """Remove the given file if present"""
60 if os.path.isfile(path):
62 print("removing", path)
66 def download(path, url, probably_big, verbose):
69 _download(path, url, probably_big, verbose, True)
72 print("\nspurious failure, trying again")
73 _download(path, url, probably_big, verbose, False)
76 def _download(path, url, probably_big, verbose, exception):
77 if probably_big or verbose:
78 print("downloading {}".format(url))
79 # see http://serverfault.com/questions/301128/how-to-download
80 if sys.platform == 'win32':
81 run(["PowerShell.exe", "/nologo", "-Command",
82 "(New-Object System.Net.WebClient)"
83 ".DownloadFile('{}', '{}')".format(url, path)],
87 if probably_big or verbose:
91 run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
96 def verify(path, sha_path, verbose):
97 """Check if the sha256 sum of the given path is valid"""
99 print("verifying", path)
100 with open(path, "rb") as source:
101 found = hashlib.sha256(source.read()).hexdigest()
102 with open(sha_path, "r") as sha256sum:
103 expected = sha256sum.readline().split()[0]
104 verified = found == expected
106 print("invalid checksum:\n"
108 " expected: {}".format(found, expected))
112 def unpack(tarball, dst, verbose=False, match=None):
113 """Unpack the given tarball file"""
114 print("extracting", tarball)
115 fname = os.path.basename(tarball).replace(".tar.gz", "")
116 with contextlib.closing(tarfile.open(tarball)) as tar:
117 for member in tar.getnames():
118 if "/" not in member:
120 name = member.replace(fname + "/", "", 1)
121 if match is not None and not name.startswith(match):
123 name = name[len(match) + 1:]
125 dst_path = os.path.join(dst, name)
127 print(" extracting", member)
128 tar.extract(member, dst)
129 src_path = os.path.join(dst, member)
130 if os.path.isdir(src_path) and os.path.exists(dst_path):
132 shutil.move(src_path, dst_path)
133 shutil.rmtree(os.path.join(dst, fname))
136 def run(args, verbose=False, exception=False, **kwargs):
137 """Run a child program in a new process"""
139 print("running: " + ' '.join(args))
141 # Use Popen here instead of call() as it apparently allows powershell on
142 # Windows to not lock up waiting for input presumably.
143 ret = subprocess.Popen(args, **kwargs)
146 err = "failed to run: " + ' '.join(args)
147 if verbose or exception:
148 raise RuntimeError(err)
152 def stage0_data(rust_root):
153 """Build a dictionary from stage0.txt"""
154 nightlies = os.path.join(rust_root, "src/stage0.txt")
155 with open(nightlies, 'r') as nightlies:
156 lines = [line.rstrip() for line in nightlies
157 if not line.startswith("#")]
158 return dict([line.split(": ", 1) for line in lines if line])
161 def format_build_time(duration):
162 """Return a nicer format for build time
164 >>> format_build_time('300')
167 return str(datetime.timedelta(seconds=int(duration)))
170 def default_build_triple():
171 """Build triple as in LLVM"""
172 default_encoding = sys.getdefaultencoding()
174 ostype = subprocess.check_output(
175 ['uname', '-s']).strip().decode(default_encoding)
176 cputype = subprocess.check_output(
177 ['uname', '-m']).strip().decode(default_encoding)
178 except (subprocess.CalledProcessError, OSError):
179 if sys.platform == 'win32':
180 return 'x86_64-pc-windows-msvc'
181 err = "uname not found"
184 # The goal here is to come up with the same triple as LLVM would,
185 # at least for the subset of platforms we're willing to target.
187 'Bitrig': 'unknown-bitrig',
188 'Darwin': 'apple-darwin',
189 'DragonFly': 'unknown-dragonfly',
190 'FreeBSD': 'unknown-freebsd',
191 'Haiku': 'unknown-haiku',
192 'NetBSD': 'unknown-netbsd',
193 'OpenBSD': 'unknown-openbsd'
196 # Consider the direct transformation first and then the special cases
197 if ostype in ostype_mapper:
198 ostype = ostype_mapper[ostype]
199 elif ostype == 'Linux':
200 os_from_sp = subprocess.check_output(
201 ['uname', '-o']).strip().decode(default_encoding)
202 if os_from_sp == 'Android':
203 ostype = 'linux-android'
205 ostype = 'unknown-linux-gnu'
206 elif ostype == 'SunOS':
207 ostype = 'sun-solaris'
208 # On Solaris, uname -m will return a machine classification instead
209 # of a cpu type, so uname -p is recommended instead. However, the
210 # output from that option is too generic for our purposes (it will
211 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
212 # must be used instead.
214 cputype = subprocess.check_output(
215 ['isainfo', '-k']).strip().decode(default_encoding)
216 except (subprocess.CalledProcessError, OSError):
217 err = "isainfo not found"
219 elif ostype.startswith('MINGW'):
220 # msys' `uname` does not print gcc configuration, but prints msys
221 # configuration. so we cannot believe `uname -m`:
222 # msys1 is always i686 and msys2 is always x86_64.
223 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
225 ostype = 'pc-windows-gnu'
227 if os.environ.get('MSYSTEM') == 'MINGW64':
229 elif ostype.startswith('MSYS'):
230 ostype = 'pc-windows-gnu'
231 elif ostype.startswith('CYGWIN_NT'):
233 if ostype.endswith('WOW64'):
235 ostype = 'pc-windows-gnu'
237 err = "unknown OS type: {}".format(ostype)
242 'aarch64': 'aarch64',
249 'powerpc': 'powerpc',
250 'powerpc64': 'powerpc64',
251 'powerpc64le': 'powerpc64le',
253 'ppc64': 'powerpc64',
254 'ppc64le': 'powerpc64le',
262 # Consider the direct transformation first and then the special cases
263 if cputype in cputype_mapper:
264 cputype = cputype_mapper[cputype]
265 elif cputype in {'xscale', 'arm'}:
267 if ostype == 'linux-android':
268 ostype = 'linux-androideabi'
269 elif cputype == 'armv6l':
271 if ostype == 'linux-android':
272 ostype = 'linux-androideabi'
275 elif cputype in {'armv7l', 'armv8l'}:
277 if ostype == 'linux-android':
278 ostype = 'linux-androideabi'
281 elif cputype == 'mips':
282 if sys.byteorder == 'big':
284 elif sys.byteorder == 'little':
287 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
288 elif cputype == 'mips64':
289 if sys.byteorder == 'big':
291 elif sys.byteorder == 'little':
294 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
295 # only the n64 ABI is supported, indicate it
297 elif cputype == 'sparcv9':
300 err = "unknown cpu type: {}".format(cputype)
303 return "{}-{}".format(cputype, ostype)
305 class RustBuild(object):
306 """Provide all the methods required to build Rust"""
308 self.cargo_channel = ''
310 self._download_url = 'https://static.rust-lang.org'
311 self.rustc_channel = ''
313 self.build_dir = os.path.join(os.getcwd(), "build")
315 self.config_toml = ''
317 self.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
318 self.use_locked_deps = ''
319 self.use_vendored_sources = ''
322 def download_stage0(self):
323 """Fetch the build system for Rust, written in Rust
325 This method will build a cache directory, then it will fetch the
326 tarball which has the stage0 compiler used to then bootstrap the Rust
329 Each downloaded tarball is extracted, after that, the script
330 will move all the content to the right place.
332 rustc_channel = self.rustc_channel
333 cargo_channel = self.cargo_channel
335 if self.rustc().startswith(self.bin_root()) and \
336 (not os.path.exists(self.rustc()) or
337 self.program_out_of_date(self.rustc_stamp())):
338 self.print_what_bootstrap_means()
339 if os.path.exists(self.bin_root()):
340 shutil.rmtree(self.bin_root())
341 filename = "rust-std-{}-{}.tar.gz".format(
342 rustc_channel, self.build)
343 pattern = "rust-std-{}".format(self.build)
344 self._download_stage0_helper(filename, pattern)
346 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
347 self._download_stage0_helper(filename, "rustc")
348 self.fix_executable("{}/bin/rustc".format(self.bin_root()))
349 self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
350 with open(self.rustc_stamp(), 'w') as rust_stamp:
351 rust_stamp.write(self.date)
353 if "pc-windows-gnu" in self.build:
354 filename = "rust-mingw-{}-{}.tar.gz".format(
355 rustc_channel, self.build)
356 self._download_stage0_helper(filename, "rust-mingw")
358 if self.cargo().startswith(self.bin_root()) and \
359 (not os.path.exists(self.cargo()) or
360 self.program_out_of_date(self.cargo_stamp())):
361 self.print_what_bootstrap_means()
362 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
363 self._download_stage0_helper(filename, "cargo")
364 self.fix_executable("{}/bin/cargo".format(self.bin_root()))
365 with open(self.cargo_stamp(), 'w') as cargo_stamp:
366 cargo_stamp.write(self.date)
368 def _download_stage0_helper(self, filename, pattern):
369 cache_dst = os.path.join(self.build_dir, "cache")
370 rustc_cache = os.path.join(cache_dst, self.date)
371 if not os.path.exists(rustc_cache):
372 os.makedirs(rustc_cache)
374 url = "{}/dist/{}".format(self._download_url, self.date)
375 tarball = os.path.join(rustc_cache, filename)
376 if not os.path.exists(tarball):
377 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
378 unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
381 def fix_executable(fname):
382 """Modifies the interpreter section of 'fname' to fix the dynamic linker
384 This method is only required on NixOS and uses the PatchELF utility to
385 change the dynamic linker of ELF executables.
387 Please see https://nixos.org/patchelf.html for more information
389 default_encoding = sys.getdefaultencoding()
391 ostype = subprocess.check_output(
392 ['uname', '-s']).strip().decode(default_encoding)
393 except subprocess.CalledProcessError:
395 except OSError as reason:
396 if getattr(reason, 'winerror', None) is not None:
400 if ostype != "Linux":
403 if not os.path.exists("/etc/NIXOS"):
405 if os.path.exists("/lib"):
408 # At this point we're pretty sure the user is running NixOS
409 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
410 print(nix_os_msg, fname)
413 interpreter = subprocess.check_output(
414 ["patchelf", "--print-interpreter", fname])
415 interpreter = interpreter.strip().decode(default_encoding)
416 except subprocess.CalledProcessError as reason:
417 print("warning: failed to call patchelf:", reason)
420 loader = interpreter.split("/")[-1]
423 ldd_output = subprocess.check_output(
424 ['ldd', '/run/current-system/sw/bin/sh'])
425 ldd_output = ldd_output.strip().decode(default_encoding)
426 except subprocess.CalledProcessError as reason:
427 print("warning: unable to call ldd:", reason)
430 for line in ldd_output.splitlines():
431 libname = line.split()[0]
432 if libname.endswith(loader):
433 loader_path = libname[:len(libname) - len(loader)]
436 print("warning: unable to find the path to the dynamic linker")
439 correct_interpreter = loader_path + loader
442 subprocess.check_output(
443 ["patchelf", "--set-interpreter", correct_interpreter, fname])
444 except subprocess.CalledProcessError as reason:
445 print("warning: failed to call patchelf:", reason)
448 def rustc_stamp(self):
449 """Return the path for .rustc-stamp
452 >>> rb.build_dir = "build"
453 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
456 return os.path.join(self.bin_root(), '.rustc-stamp')
458 def cargo_stamp(self):
459 """Return the path for .cargo-stamp
462 >>> rb.build_dir = "build"
463 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
466 return os.path.join(self.bin_root(), '.cargo-stamp')
468 def program_out_of_date(self, stamp_path):
469 """Check if the given program stamp is out of date"""
470 if not os.path.exists(stamp_path) or self.clean:
472 with open(stamp_path, 'r') as stamp:
473 return self.date != stamp.read()
476 """Return the binary root directory
479 >>> rb.build_dir = "build"
480 >>> rb.bin_root() == os.path.join("build", "stage0")
483 When the 'build' property is given should be a nested directory:
485 >>> rb.build = "devel"
486 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
489 return os.path.join(self.build_dir, self.build, "stage0")
491 def get_toml(self, key):
492 """Returns the value of the given key in config.toml, otherwise returns None
495 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
496 >>> rb.get_toml("key2")
499 If the key does not exists, the result is None:
501 >>> rb.get_toml("key3") == None
504 for line in self.config_toml.splitlines():
505 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
506 if match is not None:
507 value = match.group(1)
508 return self.get_string(value) or value.strip()
512 """Return config path for cargo"""
513 return self.program_config('cargo')
516 """Return config path for rustc"""
517 return self.program_config('rustc')
519 def program_config(self, program):
520 """Return config path for the given program
523 >>> rb.config_toml = 'rustc = "rustc"\\n'
524 >>> rb.program_config('rustc')
526 >>> rb.config_toml = ''
527 >>> cargo_path = rb.program_config('cargo')
528 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
532 config = self.get_toml(program)
535 return os.path.join(self.bin_root(), "bin", "{}{}".format(
536 program, self.exe_suffix()))
539 def get_string(line):
540 """Return the value between double quotes
542 >>> RustBuild.get_string(' "devel" ')
545 start = line.find('"')
547 end = start + 1 + line[start + 1:].find('"')
548 return line[start + 1:end]
549 start = line.find('\'')
551 end = start + 1 + line[start + 1:].find('\'')
552 return line[start + 1:end]
557 """Return a suffix for executables"""
558 if sys.platform == 'win32':
562 def print_what_bootstrap_means(self):
563 """Prints more information about the build system"""
564 if hasattr(self, 'printed'):
567 if os.path.exists(self.bootstrap_binary()):
569 if '--help' not in sys.argv or len(sys.argv) == 1:
572 print('info: the build system for Rust is written in Rust, so this')
573 print(' script is now going to download a stage0 rust compiler')
574 print(' and then compile the build system itself')
576 print('info: in the meantime you can read more about rustbuild at')
577 print(' src/bootstrap/README.md before the download finishes')
579 def bootstrap_binary(self):
580 """Return the path of the boostrap binary
583 >>> rb.build_dir = "build"
584 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
585 ... "debug", "bootstrap")
588 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
590 def build_bootstrap(self):
591 """Build bootstrap"""
592 self.print_what_bootstrap_means()
593 build_dir = os.path.join(self.build_dir, "bootstrap")
594 if self.clean and os.path.exists(build_dir):
595 shutil.rmtree(build_dir)
596 env = os.environ.copy()
597 env["RUSTC_BOOTSTRAP"] = '1'
598 env["CARGO_TARGET_DIR"] = build_dir
599 env["RUSTC"] = self.rustc()
600 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
601 (os.pathsep + env["LD_LIBRARY_PATH"]) \
602 if "LD_LIBRARY_PATH" in env else ""
603 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
604 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
605 if "DYLD_LIBRARY_PATH" in env else ""
606 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
607 (os.pathsep + env["LIBRARY_PATH"]) \
608 if "LIBRARY_PATH" in env else ""
609 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
610 os.pathsep + env["PATH"]
611 if not os.path.isfile(self.cargo()):
612 raise Exception("no cargo executable found at `{}`".format(
614 args = [self.cargo(), "build", "--manifest-path",
615 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
617 args.append("--verbose")
619 args.append("--verbose")
620 if self.use_locked_deps:
621 args.append("--locked")
622 if self.use_vendored_sources:
623 args.append("--frozen")
624 run(args, env=env, verbose=self.verbose)
626 def build_triple(self):
627 """Build triple as in LLVM"""
628 default_encoding = sys.getdefaultencoding()
629 config = self.get_toml('build')
632 return default_build_triple()
634 def update_submodules(self):
635 """Update submodules"""
636 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
637 self.get_toml('submodules') == "false":
639 print('Updating submodules')
640 default_encoding = sys.getdefaultencoding()
641 run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
642 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
643 ["git", "config", "--file",
644 os.path.join(self.rust_root, ".gitmodules"),
645 "--get-regexp", "path"]
646 ).decode(default_encoding).splitlines()]
647 submodules = [module for module in submodules
648 if not ((module.endswith("llvm") and
649 self.get_toml('llvm-config')) or
650 (module.endswith("jemalloc") and
651 self.get_toml('jemalloc')))]
652 run(["git", "submodule", "update",
653 "--init", "--recursive"] + submodules,
654 cwd=self.rust_root, verbose=self.verbose)
655 run(["git", "submodule", "-q", "foreach", "git",
656 "reset", "-q", "--hard"],
657 cwd=self.rust_root, verbose=self.verbose)
658 run(["git", "submodule", "-q", "foreach", "git",
660 cwd=self.rust_root, verbose=self.verbose)
662 def set_dev_environment(self):
663 """Set download URL for development environment"""
664 self._download_url = 'https://dev-static.rust-lang.org'
668 """Configure, fetch, build and run the initial bootstrap"""
669 parser = argparse.ArgumentParser(description='Build rust')
670 parser.add_argument('--config')
671 parser.add_argument('--build')
672 parser.add_argument('--clean', action='store_true')
673 parser.add_argument('-v', '--verbose', action='store_true')
675 args = [a for a in sys.argv if a != '-h' and a != '--help']
676 args, _ = parser.parse_known_args(args)
678 # Configure initial bootstrap
680 build.verbose = args.verbose
681 build.clean = args.clean
684 with open(args.config or 'config.toml') as config:
685 build.config_toml = config.read()
689 if '\nverbose = 2' in build.config_toml:
691 elif '\nverbose = 1' in build.config_toml:
694 build.use_vendored_sources = '\nvendor = true' in build.config_toml
696 build.use_locked_deps = '\nlocked-deps = true' in build.config_toml
698 if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
699 if os.environ.get('USER') != os.environ['SUDO_USER']:
700 build.use_vendored_sources = True
701 print('info: looks like you are running this command under `sudo`')
702 print(' and so in order to preserve your $HOME this will now')
703 print(' use vendored sources by default. Note that if this')
704 print(' does not work you should run a normal build first')
705 print(' before running a command like `sudo make install`')
707 if build.use_vendored_sources:
708 if not os.path.exists('.cargo'):
709 os.makedirs('.cargo')
710 with open('.cargo/config', 'w') as cargo_config:
711 cargo_config.write("""
713 replace-with = 'vendored-sources'
714 registry = 'https://example.com'
716 [source.vendored-sources]
717 directory = '{}/src/vendor'
718 """.format(build.rust_root))
720 if os.path.exists('.cargo'):
721 shutil.rmtree('.cargo')
723 data = stage0_data(build.rust_root)
724 build.date = data['date']
725 build.rustc_channel = data['rustc']
726 build.cargo_channel = data['cargo']
729 build.set_dev_environment()
731 build.update_submodules()
733 # Fetch/build the bootstrap
734 build.build = args.build or build.build_triple()
735 build.download_stage0()
737 build.build_bootstrap()
741 args = [build.bootstrap_binary()]
742 args.extend(sys.argv[1:])
743 env = os.environ.copy()
744 env["BUILD"] = build.build
745 env["SRC"] = build.rust_root
746 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
747 env["BOOTSTRAP_PYTHON"] = sys.executable
748 run(args, env=env, verbose=build.verbose)
752 """Entry point for the bootstrap process"""
755 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
758 if not help_triggered:
759 print("Build completed successfully in {}".format(
760 format_build_time(time() - start_time)))
761 except (SystemExit, KeyboardInterrupt) as error:
762 if hasattr(error, 'code') and isinstance(error.code, int):
763 exit_code = error.code
767 if not help_triggered:
768 print("Build completed unsuccessfully in {}".format(
769 format_build_time(time() - start_time)))
773 if __name__ == '__main__':