1 from __future__ import absolute_import, division, print_function
5 import distutils.version
20 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21 temp_path = temp_file.name
22 with tarfile.open(temp_path, "w:xz"):
25 except tarfile.CompressionError:
28 def get(base, url, path, checksums, verbose=False, do_verify=True):
29 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
30 temp_path = temp_file.name
34 if url not in checksums:
35 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
36 sha256 = checksums[url]
37 if os.path.exists(path):
38 if verify(path, sha256, False):
40 print("using already-download file", path)
44 print("ignoring already-download file",
45 path, "due to failed verification")
47 download(temp_path, "{}/{}".format(base, url), True, verbose)
48 if do_verify and not verify(temp_path, sha256, verbose):
49 raise RuntimeError("failed verification")
51 print("moving {} to {}".format(temp_path, path))
52 shutil.move(temp_path, path)
54 if os.path.isfile(temp_path):
56 print("removing", temp_path)
60 def download(path, url, probably_big, verbose):
63 _download(path, url, probably_big, verbose, True)
66 print("\nspurious failure, trying again")
67 _download(path, url, probably_big, verbose, False)
70 def _download(path, url, probably_big, verbose, exception):
71 if probably_big or verbose:
72 print("downloading {}".format(url))
73 # see https://serverfault.com/questions/301128/how-to-download
74 if sys.platform == 'win32':
75 run(["PowerShell.exe", "/nologo", "-Command",
76 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
77 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
81 if probably_big or verbose:
85 require(["curl", "--version"])
87 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
88 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
89 "--retry", "3", "-Sf", "-o", path, url],
94 def verify(path, expected, verbose):
95 """Check if the sha256 sum of the given path is valid"""
97 print("verifying", path)
98 with open(path, "rb") as source:
99 found = hashlib.sha256(source.read()).hexdigest()
100 verified = found == expected
102 print("invalid checksum:\n"
104 " expected: {}".format(found, expected))
108 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
109 """Unpack the given tarball file"""
110 print("extracting", tarball)
111 fname = os.path.basename(tarball).replace(tarball_suffix, "")
112 with contextlib.closing(tarfile.open(tarball)) as tar:
113 for member in tar.getnames():
114 if "/" not in member:
116 name = member.replace(fname + "/", "", 1)
117 if match is not None and not name.startswith(match):
119 name = name[len(match) + 1:]
121 dst_path = os.path.join(dst, name)
123 print(" extracting", member)
124 tar.extract(member, dst)
125 src_path = os.path.join(dst, member)
126 if os.path.isdir(src_path) and os.path.exists(dst_path):
128 shutil.move(src_path, dst_path)
129 shutil.rmtree(os.path.join(dst, fname))
132 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
133 """Run a child program in a new process"""
135 print("running: " + ' '.join(args))
137 # Use Popen here instead of call() as it apparently allows powershell on
138 # Windows to not lock up waiting for input presumably.
139 ret = subprocess.Popen(args, **kwargs)
142 err = "failed to run: " + ' '.join(args)
143 if verbose or exception:
144 raise RuntimeError(err)
145 # For most failures, we definitely do want to print this error, or the user will have no
146 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
147 # have already printed an error above, so there's no need to print the exact command we're
155 def require(cmd, exit=True):
156 '''Run a command, returning its output.
158 If `exit` is `True`, exit the process.
159 Otherwise, return None.'''
161 return subprocess.check_output(cmd).strip()
162 except (subprocess.CalledProcessError, OSError) as exc:
165 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
166 print("Please make sure it's installed and in the path.")
170 def format_build_time(duration):
171 """Return a nicer format for build time
173 >>> format_build_time('300')
176 return str(datetime.timedelta(seconds=int(duration)))
179 def default_build_triple(verbose):
180 """Build triple as in LLVM"""
181 # If the user already has a host build triple with an existing `rustc`
182 # install, use their preference. This fixes most issues with Windows builds
183 # being detected as GNU instead of MSVC.
184 default_encoding = sys.getdefaultencoding()
186 version = subprocess.check_output(["rustc", "--version", "--verbose"],
187 stderr=subprocess.DEVNULL)
188 version = version.decode(default_encoding)
189 host = next(x for x in version.split('\n') if x.startswith("host: "))
190 triple = host.split("host: ")[1]
192 print("detected default triple {}".format(triple))
194 except Exception as e:
196 print("rustup not detected: {}".format(e))
197 print("falling back to auto-detect")
199 required = sys.platform != 'win32'
200 ostype = require(["uname", "-s"], exit=required)
201 cputype = require(['uname', '-m'], exit=required)
203 # If we do not have `uname`, assume Windows.
204 if ostype is None or cputype is None:
205 return 'x86_64-pc-windows-msvc'
207 ostype = ostype.decode(default_encoding)
208 cputype = cputype.decode(default_encoding)
210 # The goal here is to come up with the same triple as LLVM would,
211 # at least for the subset of platforms we're willing to target.
213 'Darwin': 'apple-darwin',
214 'DragonFly': 'unknown-dragonfly',
215 'FreeBSD': 'unknown-freebsd',
216 'Haiku': 'unknown-haiku',
217 'NetBSD': 'unknown-netbsd',
218 'OpenBSD': 'unknown-openbsd'
221 # Consider the direct transformation first and then the special cases
222 if ostype in ostype_mapper:
223 ostype = ostype_mapper[ostype]
224 elif ostype == 'Linux':
225 os_from_sp = subprocess.check_output(
226 ['uname', '-o']).strip().decode(default_encoding)
227 if os_from_sp == 'Android':
228 ostype = 'linux-android'
230 ostype = 'unknown-linux-gnu'
231 elif ostype == 'SunOS':
232 ostype = 'pc-solaris'
233 # On Solaris, uname -m will return a machine classification instead
234 # of a cpu type, so uname -p is recommended instead. However, the
235 # output from that option is too generic for our purposes (it will
236 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
237 # must be used instead.
238 cputype = require(['isainfo', '-k']).decode(default_encoding)
239 # sparc cpus have sun as a target vendor
240 if 'sparc' in cputype:
241 ostype = 'sun-solaris'
242 elif ostype.startswith('MINGW'):
243 # msys' `uname` does not print gcc configuration, but prints msys
244 # configuration. so we cannot believe `uname -m`:
245 # msys1 is always i686 and msys2 is always x86_64.
246 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
248 ostype = 'pc-windows-gnu'
250 if os.environ.get('MSYSTEM') == 'MINGW64':
252 elif ostype.startswith('MSYS'):
253 ostype = 'pc-windows-gnu'
254 elif ostype.startswith('CYGWIN_NT'):
256 if ostype.endswith('WOW64'):
258 ostype = 'pc-windows-gnu'
259 elif sys.platform == 'win32':
260 # Some Windows platforms might have a `uname` command that returns a
261 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
262 # these cases, fall back to using sys.platform.
263 return 'x86_64-pc-windows-msvc'
265 err = "unknown OS type: {}".format(ostype)
268 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
269 cputype = subprocess.check_output(
270 ['uname', '-p']).strip().decode(default_encoding)
273 'aarch64': 'aarch64',
281 'powerpc': 'powerpc',
282 'powerpc64': 'powerpc64',
283 'powerpc64le': 'powerpc64le',
285 'ppc64': 'powerpc64',
286 'ppc64le': 'powerpc64le',
287 'riscv64': 'riscv64gc',
295 # Consider the direct transformation first and then the special cases
296 if cputype in cputype_mapper:
297 cputype = cputype_mapper[cputype]
298 elif cputype in {'xscale', 'arm'}:
300 if ostype == 'linux-android':
301 ostype = 'linux-androideabi'
302 elif ostype == 'unknown-freebsd':
303 cputype = subprocess.check_output(
304 ['uname', '-p']).strip().decode(default_encoding)
305 ostype = 'unknown-freebsd'
306 elif cputype == 'armv6l':
308 if ostype == 'linux-android':
309 ostype = 'linux-androideabi'
312 elif cputype in {'armv7l', 'armv8l'}:
314 if ostype == 'linux-android':
315 ostype = 'linux-androideabi'
318 elif cputype == 'mips':
319 if sys.byteorder == 'big':
321 elif sys.byteorder == 'little':
324 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
325 elif cputype == 'mips64':
326 if sys.byteorder == 'big':
328 elif sys.byteorder == 'little':
331 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
332 # only the n64 ABI is supported, indicate it
334 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
337 err = "unknown cpu type: {}".format(cputype)
340 return "{}-{}".format(cputype, ostype)
343 @contextlib.contextmanager
344 def output(filepath):
345 tmp = filepath + '.tmp'
346 with open(tmp, 'w') as f:
349 if os.path.exists(filepath):
350 os.remove(filepath) # PermissionError/OSError on Win32 if in use
352 shutil.copy2(tmp, filepath)
355 os.rename(tmp, filepath)
358 class Stage0Toolchain:
359 def __init__(self, stage0_payload):
360 self.date = stage0_payload["date"]
361 self.version = stage0_payload["version"]
364 return self.version + "-" + self.date
367 class RustBuild(object):
368 """Provide all the methods required to build Rust"""
370 self.checksums_sha256 = {}
371 self.stage0_compiler = None
372 self.stage0_rustfmt = None
373 self._download_url = ''
377 self.config_toml = ''
379 self.use_locked_deps = ''
380 self.use_vendored_sources = ''
382 self.git_version = None
383 self.nix_deps_dir = None
384 self.rustc_commit = None
386 def download_toolchain(self, stage0=True, rustc_channel=None):
387 """Fetch the build system for Rust, written in Rust
389 This method will build a cache directory, then it will fetch the
390 tarball which has the stage0 compiler used to then bootstrap the Rust
393 Each downloaded tarball is extracted, after that, the script
394 will move all the content to the right place.
396 if rustc_channel is None:
397 rustc_channel = self.stage0_compiler.version
398 bin_root = self.bin_root(stage0)
400 key = self.stage0_compiler.date
402 key += str(self.rustc_commit)
403 if self.rustc(stage0).startswith(bin_root) and \
404 (not os.path.exists(self.rustc(stage0)) or
405 self.program_out_of_date(self.rustc_stamp(stage0), key)):
406 if os.path.exists(bin_root):
407 shutil.rmtree(bin_root)
408 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
409 filename = "rust-std-{}-{}{}".format(
410 rustc_channel, self.build, tarball_suffix)
411 pattern = "rust-std-{}".format(self.build)
412 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
413 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
415 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
416 # download-rustc doesn't need its own cargo, it can just use beta's.
418 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
420 self._download_component_helper(filename, "cargo", tarball_suffix)
421 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
423 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
424 self._download_component_helper(
425 filename, "rustc-dev", tarball_suffix, stage0
428 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
429 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
430 lib_dir = "{}/lib".format(bin_root)
431 for lib in os.listdir(lib_dir):
432 if lib.endswith(".so"):
433 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
434 with output(self.rustc_stamp(stage0)) as rust_stamp:
435 rust_stamp.write(key)
437 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
438 not os.path.exists(self.rustfmt())
439 or self.program_out_of_date(
440 self.rustfmt_stamp(),
441 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
444 if self.stage0_rustfmt is not None:
445 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
446 filename = "rustfmt-{}-{}{}".format(
447 self.stage0_rustfmt.version, self.build, tarball_suffix,
449 self._download_component_helper(
450 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
452 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
453 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
454 with output(self.rustfmt_stamp()) as rustfmt_stamp:
455 rustfmt_stamp.write(self.stage0_rustfmt.channel())
457 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
458 if self.downloading_llvm() and stage0:
459 # We want the most recent LLVM submodule update to avoid downloading
460 # LLVM more often than necessary.
462 # This git command finds that commit SHA, looking for bors-authored
463 # commits that modified src/llvm-project or other relevant version
466 # This works even in a repository that has not yet initialized
468 top_level = subprocess.check_output([
469 "git", "rev-parse", "--show-toplevel",
470 ]).decode(sys.getdefaultencoding()).strip()
471 llvm_sha = subprocess.check_output([
472 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
473 "--first-parent", "HEAD",
475 "{}/src/llvm-project".format(top_level),
476 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
477 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
478 "{}/src/version".format(top_level)
479 ]).decode(sys.getdefaultencoding()).strip()
480 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
481 llvm_root = self.llvm_root()
482 llvm_lib = os.path.join(llvm_root, "lib")
483 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
484 self._download_ci_llvm(llvm_sha, llvm_assertions)
485 for binary in ["llvm-config", "FileCheck"]:
486 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
487 for lib in os.listdir(llvm_lib):
488 if lib.endswith(".so"):
489 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
490 with output(self.llvm_stamp()) as llvm_stamp:
491 llvm_stamp.write(llvm_sha + str(llvm_assertions))
493 def downloading_llvm(self):
494 opt = self.get_toml('download-ci-llvm', 'llvm')
495 # This is currently all tier 1 targets (since others may not have CI
497 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
498 supported_platforms = [
499 "aarch64-unknown-linux-gnu",
500 "i686-pc-windows-gnu",
501 "i686-pc-windows-msvc",
502 "i686-unknown-linux-gnu",
503 "x86_64-unknown-linux-gnu",
504 "x86_64-apple-darwin",
505 "x86_64-pc-windows-gnu",
506 "x86_64-pc-windows-msvc",
508 return opt == "true" \
509 or (opt == "if-available" and self.build in supported_platforms)
511 def _download_component_helper(
512 self, filename, pattern, tarball_suffix, stage0=True, key=None
516 key = self.stage0_compiler.date
518 key = self.rustc_commit
519 cache_dst = os.path.join(self.build_dir, "cache")
520 rustc_cache = os.path.join(cache_dst, key)
521 if not os.path.exists(rustc_cache):
522 os.makedirs(rustc_cache)
525 base = self._download_url
526 url = "dist/{}".format(key)
528 base = "https://ci-artifacts.rust-lang.org"
529 url = "rustc-builds/{}".format(self.rustc_commit)
530 tarball = os.path.join(rustc_cache, filename)
531 if not os.path.exists(tarball):
534 "{}/{}".format(url, filename),
536 self.checksums_sha256,
537 verbose=self.verbose,
540 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
542 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
544 print("error: could not find commit hash for downloading LLVM")
545 print("help: maybe your repository history is too shallow?")
546 print("help: consider disabling `download-ci-llvm`")
547 print("help: or fetch enough history to include one upstream commit")
549 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
550 cache_dst = os.path.join(self.build_dir, "cache")
551 rustc_cache = os.path.join(cache_dst, cache_prefix)
552 if not os.path.exists(rustc_cache):
553 os.makedirs(rustc_cache)
555 base = "https://ci-artifacts.rust-lang.org"
556 url = "rustc-builds/{}".format(llvm_sha)
558 url = url.replace('rustc-builds', 'rustc-builds-alt')
559 # ci-artifacts are only stored as .xz, not .gz
561 print("error: XZ support is required to download LLVM")
562 print("help: consider disabling `download-ci-llvm` or using python3")
564 tarball_suffix = '.tar.xz'
565 filename = "rust-dev-nightly-" + self.build + tarball_suffix
566 tarball = os.path.join(rustc_cache, filename)
567 if not os.path.exists(tarball):
570 "{}/{}".format(url, filename),
572 self.checksums_sha256,
573 verbose=self.verbose,
576 unpack(tarball, tarball_suffix, self.llvm_root(),
578 verbose=self.verbose)
580 def fix_bin_or_dylib(self, fname):
581 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
582 or the RPATH section, to fix the dynamic library search path
584 This method is only required on NixOS and uses the PatchELF utility to
585 change the interpreter/RPATH of ELF executables.
587 Please see https://nixos.org/patchelf.html for more information
589 default_encoding = sys.getdefaultencoding()
591 ostype = subprocess.check_output(
592 ['uname', '-s']).strip().decode(default_encoding)
593 except subprocess.CalledProcessError:
595 except OSError as reason:
596 if getattr(reason, 'winerror', None) is not None:
600 if ostype != "Linux":
603 # If the user has asked binaries to be patched for Nix, then
604 # don't check for NixOS or `/lib`, just continue to the patching.
605 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
606 # Use `/etc/os-release` instead of `/etc/NIXOS`.
607 # The latter one does not exist on NixOS when using tmpfs as root.
609 with open("/etc/os-release", "r") as f:
610 if not any(line.strip() == "ID=nixos" for line in f):
612 except FileNotFoundError:
614 if os.path.exists("/lib"):
617 # At this point we're pretty sure the user is running NixOS or
619 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
620 print(nix_os_msg, fname)
622 # Only build `.nix-deps` once.
623 nix_deps_dir = self.nix_deps_dir
625 # Run `nix-build` to "build" each dependency (which will likely reuse
626 # the existing `/nix/store` copy, or at most download a pre-built copy).
628 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
629 # directory, but still reference the actual `/nix/store` path in the rpath
630 # as it makes it significantly more robust against changes to the location of
631 # the `.nix-deps` location.
633 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
634 # zlib: Needed as a system dependency of `libLLVM-*.so`.
635 # patchelf: Needed for patching ELF binaries (see doc comment above).
636 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
638 with (import <nixpkgs> {});
640 name = "rust-stage0-dependencies";
649 subprocess.check_output([
650 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
652 except subprocess.CalledProcessError as reason:
653 print("warning: failed to call nix-build:", reason)
655 self.nix_deps_dir = nix_deps_dir
657 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
659 # Relative default, all binary and dynamic libraries we ship
660 # appear to have this (even when `../lib` is redundant).
662 os.path.join(os.path.realpath(nix_deps_dir), "lib")
664 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
665 if not fname.endswith(".so"):
666 # Finally, set the corret .interp for binaries
667 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
668 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
671 subprocess.check_output([patchelf] + patchelf_args + [fname])
672 except subprocess.CalledProcessError as reason:
673 print("warning: failed to call patchelf:", reason)
676 # If `download-rustc` is set, download the most recent commit with CI artifacts
677 def maybe_download_ci_toolchain(self):
678 # If `download-rustc` is not set, default to rebuilding.
679 download_rustc = self.get_toml("download-rustc", section="rust")
680 if download_rustc is None or download_rustc == "false":
682 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
684 # Handle running from a directory other than the top level
685 rev_parse = ["git", "rev-parse", "--show-toplevel"]
686 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
687 compiler = "{}/compiler/".format(top_level)
688 library = "{}/library/".format(top_level)
690 # Look for a version to compare to based on the current commit.
691 # Only commits merged by bors will have CI artifacts.
693 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
694 "--first-parent", "HEAD"
696 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
698 print("error: could not find commit hash for downloading rustc")
699 print("help: maybe your repository history is too shallow?")
700 print("help: consider disabling `download-rustc`")
701 print("help: or fetch enough history to include one upstream commit")
704 # Warn if there were changes to the compiler or standard library since the ancestor commit.
705 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
707 if download_rustc == "if-unchanged":
709 print("warning: `download-rustc` is enabled, but there are changes to \
710 compiler/ or library/")
713 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
714 self.rustc_commit = commit
715 # FIXME: support downloading artifacts from the beta channel
716 self.download_toolchain(False, "nightly")
718 def rustc_stamp(self, stage0):
719 """Return the path for .rustc-stamp at the given stage
722 >>> rb.build_dir = "build"
723 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
725 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
728 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
730 def rustfmt_stamp(self):
731 """Return the path for .rustfmt-stamp
734 >>> rb.build_dir = "build"
735 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
738 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
740 def llvm_stamp(self):
741 """Return the path for .rustfmt-stamp
744 >>> rb.build_dir = "build"
745 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
748 return os.path.join(self.llvm_root(), '.llvm-stamp')
751 def program_out_of_date(self, stamp_path, key):
752 """Check if the given program stamp is out of date"""
753 if not os.path.exists(stamp_path) or self.clean:
755 with open(stamp_path, 'r') as stamp:
756 return key != stamp.read()
758 def bin_root(self, stage0):
759 """Return the binary root directory for the given stage
762 >>> rb.build_dir = "build"
763 >>> rb.bin_root(True) == os.path.join("build", "stage0")
765 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
768 When the 'build' property is given should be a nested directory:
770 >>> rb.build = "devel"
771 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
778 return os.path.join(self.build_dir, self.build, subdir)
781 """Return the CI LLVM root directory
784 >>> rb.build_dir = "build"
785 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
788 When the 'build' property is given should be a nested directory:
790 >>> rb.build = "devel"
791 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
794 return os.path.join(self.build_dir, self.build, "ci-llvm")
796 def get_toml(self, key, section=None):
797 """Returns the value of the given key in config.toml, otherwise returns None
800 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
801 >>> rb.get_toml("key2")
804 If the key does not exists, the result is None:
806 >>> rb.get_toml("key3") is None
809 Optionally also matches the section the key appears in
811 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
812 >>> rb.get_toml('key', 'a')
814 >>> rb.get_toml('key', 'b')
816 >>> rb.get_toml('key', 'c') is None
819 >>> rb.config_toml = 'key1 = true'
820 >>> rb.get_toml("key1")
825 for line in self.config_toml.splitlines():
826 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
827 if section_match is not None:
828 cur_section = section_match.group(1)
830 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
831 if match is not None:
832 value = match.group(1)
833 if section is None or section == cur_section:
834 return self.get_string(value) or value.strip()
838 """Return config path for cargo"""
839 return self.program_config('cargo')
841 def rustc(self, stage0):
842 """Return config path for rustc"""
843 return self.program_config('rustc', stage0)
846 """Return config path for rustfmt"""
847 if self.stage0_rustfmt is None:
849 return self.program_config('rustfmt')
851 def program_config(self, program, stage0=True):
852 """Return config path for the given program at the given stage
855 >>> rb.config_toml = 'rustc = "rustc"\\n'
856 >>> rb.program_config('rustc')
858 >>> rb.config_toml = ''
859 >>> cargo_path = rb.program_config('cargo', True)
860 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
863 >>> cargo_path = rb.program_config('cargo', False)
864 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
868 config = self.get_toml(program)
870 return os.path.expanduser(config)
871 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
872 program, self.exe_suffix()))
875 def get_string(line):
876 """Return the value between double quotes
878 >>> RustBuild.get_string(' "devel" ')
880 >>> RustBuild.get_string(" 'devel' ")
882 >>> RustBuild.get_string('devel') is None
884 >>> RustBuild.get_string(' "devel ')
887 start = line.find('"')
889 end = start + 1 + line[start + 1:].find('"')
890 return line[start + 1:end]
891 start = line.find('\'')
893 end = start + 1 + line[start + 1:].find('\'')
894 return line[start + 1:end]
899 """Return a suffix for executables"""
900 if sys.platform == 'win32':
904 def bootstrap_binary(self):
905 """Return the path of the bootstrap binary
908 >>> rb.build_dir = "build"
909 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
910 ... "debug", "bootstrap")
913 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
915 def build_bootstrap(self):
916 """Build bootstrap"""
917 build_dir = os.path.join(self.build_dir, "bootstrap")
918 if self.clean and os.path.exists(build_dir):
919 shutil.rmtree(build_dir)
920 env = os.environ.copy()
921 # `CARGO_BUILD_TARGET` breaks bootstrap build.
922 # See also: <https://github.com/rust-lang/rust/issues/70208>.
923 if "CARGO_BUILD_TARGET" in env:
924 del env["CARGO_BUILD_TARGET"]
925 env["CARGO_TARGET_DIR"] = build_dir
926 env["RUSTC"] = self.rustc(True)
927 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
928 (os.pathsep + env["LD_LIBRARY_PATH"]) \
929 if "LD_LIBRARY_PATH" in env else ""
930 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
931 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
932 if "DYLD_LIBRARY_PATH" in env else ""
933 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
934 (os.pathsep + env["LIBRARY_PATH"]) \
935 if "LIBRARY_PATH" in env else ""
937 # preserve existing RUSTFLAGS
938 env.setdefault("RUSTFLAGS", "")
939 build_section = "target.{}".format(self.build)
941 if self.get_toml("crt-static", build_section) == "true":
942 target_features += ["+crt-static"]
943 elif self.get_toml("crt-static", build_section) == "false":
944 target_features += ["-crt-static"]
946 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
947 target_linker = self.get_toml("linker", build_section)
948 if target_linker is not None:
949 env["RUSTFLAGS"] += " -C linker=" + target_linker
950 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
951 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
952 if self.get_toml("deny-warnings", "rust") != "false":
953 env["RUSTFLAGS"] += " -Dwarnings"
955 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
956 os.pathsep + env["PATH"]
957 if not os.path.isfile(self.cargo()):
958 raise Exception("no cargo executable found at `{}`".format(
960 args = [self.cargo(), "build", "--manifest-path",
961 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
962 for _ in range(1, self.verbose):
963 args.append("--verbose")
964 if self.use_locked_deps:
965 args.append("--locked")
966 if self.use_vendored_sources:
967 args.append("--frozen")
968 run(args, env=env, verbose=self.verbose)
970 def build_triple(self):
971 """Build triple as in LLVM
973 Note that `default_build_triple` is moderately expensive,
974 so use `self.build` where possible.
976 config = self.get_toml('build')
979 return default_build_triple(self.verbose)
981 def check_submodule(self, module, slow_submodules):
982 if not slow_submodules:
983 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
984 cwd=os.path.join(self.rust_root, module),
985 stdout=subprocess.PIPE)
990 def update_submodule(self, module, checked_out, recorded_submodules):
991 module_path = os.path.join(self.rust_root, module)
993 if checked_out is not None:
994 default_encoding = sys.getdefaultencoding()
995 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
996 if recorded_submodules[module] == checked_out:
999 print("Updating submodule", module)
1001 run(["git", "submodule", "-q", "sync", module],
1002 cwd=self.rust_root, verbose=self.verbose)
1004 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1005 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1006 update_args.append("--progress")
1007 update_args.append(module)
1008 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1010 run(["git", "reset", "-q", "--hard"],
1011 cwd=module_path, verbose=self.verbose)
1012 run(["git", "clean", "-qdfx"],
1013 cwd=module_path, verbose=self.verbose)
1015 def update_submodules(self):
1016 """Update submodules"""
1017 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1018 self.get_toml('submodules') == "false":
1021 default_encoding = sys.getdefaultencoding()
1023 # check the existence and version of 'git' command
1024 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1025 self.git_version = distutils.version.LooseVersion(git_version_str)
1027 slow_submodules = self.get_toml('fast-submodules') == "false"
1030 print('Unconditionally updating submodules')
1032 print('Updating only changed submodules')
1033 default_encoding = sys.getdefaultencoding()
1034 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1035 # currently requires everything in a workspace to be "locally present" when starting a
1036 # build, and will give a hard error if any Cargo.toml files are missing.
1037 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1038 # share a workspace with any tools - maybe it could be excluded from the workspace?
1039 # That will still require cloning the submodules the second you check the standard
1040 # library, though...
1041 # FIXME: Is there a way to avoid hard-coding the submodules required?
1042 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1044 "src/tools/rust-installer",
1048 "library/backtrace",
1051 filtered_submodules = []
1052 submodules_names = []
1053 for module in submodules:
1054 check = self.check_submodule(module, slow_submodules)
1055 filtered_submodules.append((module, check))
1056 submodules_names.append(module)
1057 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1058 cwd=self.rust_root, stdout=subprocess.PIPE)
1059 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1060 # { filename: hash }
1061 recorded_submodules = {}
1062 for data in recorded:
1063 # [mode, kind, hash, filename]
1065 recorded_submodules[data[3]] = data[2]
1066 for module in filtered_submodules:
1067 self.update_submodule(module[0], module[1], recorded_submodules)
1068 print("Submodules updated in %.2f seconds" % (time() - start_time))
1070 def set_dist_environment(self, url):
1071 """Set download URL for normal environment"""
1072 if 'RUSTUP_DIST_SERVER' in os.environ:
1073 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1075 self._download_url = url
1077 def check_vendored_status(self):
1078 """Check that vendoring is configured properly"""
1079 vendor_dir = os.path.join(self.rust_root, 'vendor')
1080 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1081 if os.environ.get('USER') != os.environ['SUDO_USER']:
1082 self.use_vendored_sources = True
1083 print('info: looks like you are running this command under `sudo`')
1084 print(' and so in order to preserve your $HOME this will now')
1085 print(' use vendored sources by default.')
1086 if not os.path.exists(vendor_dir):
1087 print('error: vendoring required, but vendor directory does not exist.')
1088 print(' Run `cargo vendor` without sudo to initialize the '
1089 'vendor directory.')
1090 raise Exception("{} not found".format(vendor_dir))
1092 if self.use_vendored_sources:
1093 if not os.path.exists('.cargo'):
1094 os.makedirs('.cargo')
1095 with output('.cargo/config') as cargo_config:
1097 "[source.crates-io]\n"
1098 "replace-with = 'vendored-sources'\n"
1099 "registry = 'https://example.com'\n"
1101 "[source.vendored-sources]\n"
1102 "directory = '{}/vendor'\n"
1103 .format(self.rust_root))
1105 if os.path.exists('.cargo'):
1106 shutil.rmtree('.cargo')
1108 def ensure_vendored(self):
1109 """Ensure that the vendored sources are available if needed"""
1110 vendor_dir = os.path.join(self.rust_root, 'vendor')
1111 # Note that this does not handle updating the vendored dependencies if
1112 # the rust git repository is updated. Normal development usually does
1113 # not use vendoring, so hopefully this isn't too much of a problem.
1114 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1118 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1119 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1120 ], verbose=self.verbose, cwd=self.rust_root)
1123 def bootstrap(help_triggered):
1124 """Configure, fetch, build and run the initial bootstrap"""
1126 # If the user is asking for help, let them know that the whole download-and-build
1127 # process has to happen before anything is printed out.
1129 print("info: Downloading and building bootstrap before processing --help")
1130 print(" command. See src/bootstrap/README.md for help with common")
1133 parser = argparse.ArgumentParser(description='Build rust')
1134 parser.add_argument('--config')
1135 parser.add_argument('--build')
1136 parser.add_argument('--clean', action='store_true')
1137 parser.add_argument('-v', '--verbose', action='count', default=0)
1139 args = [a for a in sys.argv if a != '-h' and a != '--help']
1140 args, _ = parser.parse_known_args(args)
1142 # Configure initial bootstrap
1144 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1145 build.verbose = args.verbose
1146 build.clean = args.clean
1148 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1150 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1151 if not toml_path and os.path.exists('config.toml'):
1152 toml_path = 'config.toml'
1155 if not os.path.exists(toml_path):
1156 toml_path = os.path.join(build.rust_root, toml_path)
1158 with open(toml_path) as config:
1159 build.config_toml = config.read()
1161 profile = build.get_toml('profile')
1162 if profile is not None:
1163 include_file = 'config.{}.toml'.format(profile)
1164 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1165 include_path = os.path.join(include_dir, include_file)
1166 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1167 # specific key, so appending our defaults at the end allows the user to override them
1168 with open(include_path) as included_toml:
1169 build.config_toml += os.linesep + included_toml.read()
1171 config_verbose = build.get_toml('verbose', 'build')
1172 if config_verbose is not None:
1173 build.verbose = max(build.verbose, int(config_verbose))
1175 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1177 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1179 build.check_vendored_status()
1181 build_dir = build.get_toml('build-dir', 'build') or 'build'
1182 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1184 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1186 build.checksums_sha256 = data["checksums_sha256"]
1187 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1188 if data.get("rustfmt") is not None:
1189 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1191 build.set_dist_environment(data["dist_server"])
1193 build.build = args.build or build.build_triple()
1194 build.update_submodules()
1196 # Fetch/build the bootstrap
1197 build.download_toolchain()
1198 # Download the master compiler if `download-rustc` is set
1199 build.maybe_download_ci_toolchain()
1201 build.ensure_vendored()
1202 build.build_bootstrap()
1206 args = [build.bootstrap_binary()]
1207 args.extend(sys.argv[1:])
1208 env = os.environ.copy()
1209 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1210 env["BOOTSTRAP_PYTHON"] = sys.executable
1211 env["BUILD_DIR"] = build.build_dir
1212 env["RUSTC_BOOTSTRAP"] = '1'
1214 env["BOOTSTRAP_CONFIG"] = toml_path
1215 if build.rustc_commit is not None:
1216 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1217 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1221 """Entry point for the bootstrap process"""
1224 # x.py help <cmd> ...
1225 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1226 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1229 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1231 bootstrap(help_triggered)
1232 if not help_triggered:
1233 print("Build completed successfully in {}".format(
1234 format_build_time(time() - start_time)))
1235 except (SystemExit, KeyboardInterrupt) as error:
1236 if hasattr(error, 'code') and isinstance(error.code, int):
1237 exit_code = error.code
1241 if not help_triggered:
1242 print("Build completed unsuccessfully in {}".format(
1243 format_build_time(time() - start_time)))
1247 if __name__ == '__main__':