1 from __future__ import absolute_import, division, print_function
5 import distutils.version
20 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21 temp_path = temp_file.name
22 with tarfile.open(temp_path, "w:xz"):
25 except tarfile.CompressionError:
28 def get(base, url, path, checksums, verbose=False, do_verify=True):
29 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
30 temp_path = temp_file.name
34 if url not in checksums:
35 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
36 sha256 = checksums[url]
37 if os.path.exists(path):
38 if verify(path, sha256, False):
40 print("using already-download file", path)
44 print("ignoring already-download file",
45 path, "due to failed verification")
47 download(temp_path, "{}/{}".format(base, url), True, verbose)
48 if do_verify and not verify(temp_path, sha256, verbose):
49 raise RuntimeError("failed verification")
51 print("moving {} to {}".format(temp_path, path))
52 shutil.move(temp_path, path)
54 if os.path.isfile(temp_path):
56 print("removing", temp_path)
60 def download(path, url, probably_big, verbose):
63 _download(path, url, probably_big, verbose, True)
66 print("\nspurious failure, trying again")
67 _download(path, url, probably_big, verbose, False)
70 def _download(path, url, probably_big, verbose, exception):
71 if probably_big or verbose:
72 print("downloading {}".format(url))
73 # see https://serverfault.com/questions/301128/how-to-download
74 if sys.platform == 'win32':
75 run(["PowerShell.exe", "/nologo", "-Command",
76 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
77 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
81 if probably_big or verbose:
85 require(["curl", "--version"])
87 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
88 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
89 "--retry", "3", "-Sf", "-o", path, url],
94 def verify(path, expected, verbose):
95 """Check if the sha256 sum of the given path is valid"""
97 print("verifying", path)
98 with open(path, "rb") as source:
99 found = hashlib.sha256(source.read()).hexdigest()
100 verified = found == expected
102 print("invalid checksum:\n"
104 " expected: {}".format(found, expected))
108 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
109 """Unpack the given tarball file"""
110 print("extracting", tarball)
111 fname = os.path.basename(tarball).replace(tarball_suffix, "")
112 with contextlib.closing(tarfile.open(tarball)) as tar:
113 for member in tar.getnames():
114 if "/" not in member:
116 name = member.replace(fname + "/", "", 1)
117 if match is not None and not name.startswith(match):
119 name = name[len(match) + 1:]
121 dst_path = os.path.join(dst, name)
123 print(" extracting", member)
124 tar.extract(member, dst)
125 src_path = os.path.join(dst, member)
126 if os.path.isdir(src_path) and os.path.exists(dst_path):
128 shutil.move(src_path, dst_path)
129 shutil.rmtree(os.path.join(dst, fname))
132 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
133 """Run a child program in a new process"""
135 print("running: " + ' '.join(args))
137 # Use Popen here instead of call() as it apparently allows powershell on
138 # Windows to not lock up waiting for input presumably.
139 ret = subprocess.Popen(args, **kwargs)
142 err = "failed to run: " + ' '.join(args)
143 if verbose or exception:
144 raise RuntimeError(err)
145 # For most failures, we definitely do want to print this error, or the user will have no
146 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
147 # have already printed an error above, so there's no need to print the exact command we're
155 def require(cmd, exit=True):
156 '''Run a command, returning its output.
158 If `exit` is `True`, exit the process.
159 Otherwise, return None.'''
161 return subprocess.check_output(cmd).strip()
162 except (subprocess.CalledProcessError, OSError) as exc:
165 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
166 print("Please make sure it's installed and in the path.")
170 def format_build_time(duration):
171 """Return a nicer format for build time
173 >>> format_build_time('300')
176 return str(datetime.timedelta(seconds=int(duration)))
179 def default_build_triple(verbose):
180 """Build triple as in LLVM"""
181 # If the user already has a host build triple with an existing `rustc`
182 # install, use their preference. This fixes most issues with Windows builds
183 # being detected as GNU instead of MSVC.
184 default_encoding = sys.getdefaultencoding()
186 version = subprocess.check_output(["rustc", "--version", "--verbose"],
187 stderr=subprocess.DEVNULL)
188 version = version.decode(default_encoding)
189 host = next(x for x in version.split('\n') if x.startswith("host: "))
190 triple = host.split("host: ")[1]
192 print("detected default triple {}".format(triple))
194 except Exception as e:
196 print("rustup not detected: {}".format(e))
197 print("falling back to auto-detect")
199 required = sys.platform != 'win32'
200 ostype = require(["uname", "-s"], exit=required)
201 cputype = require(['uname', '-m'], exit=required)
203 # If we do not have `uname`, assume Windows.
204 if ostype is None or cputype is None:
205 return 'x86_64-pc-windows-msvc'
207 ostype = ostype.decode(default_encoding)
208 cputype = cputype.decode(default_encoding)
210 # The goal here is to come up with the same triple as LLVM would,
211 # at least for the subset of platforms we're willing to target.
213 'Darwin': 'apple-darwin',
214 'DragonFly': 'unknown-dragonfly',
215 'FreeBSD': 'unknown-freebsd',
216 'Haiku': 'unknown-haiku',
217 'NetBSD': 'unknown-netbsd',
218 'OpenBSD': 'unknown-openbsd'
221 # Consider the direct transformation first and then the special cases
222 if ostype in ostype_mapper:
223 ostype = ostype_mapper[ostype]
224 elif ostype == 'Linux':
225 os_from_sp = subprocess.check_output(
226 ['uname', '-o']).strip().decode(default_encoding)
227 if os_from_sp == 'Android':
228 ostype = 'linux-android'
230 ostype = 'unknown-linux-gnu'
231 elif ostype == 'SunOS':
232 ostype = 'pc-solaris'
233 # On Solaris, uname -m will return a machine classification instead
234 # of a cpu type, so uname -p is recommended instead. However, the
235 # output from that option is too generic for our purposes (it will
236 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
237 # must be used instead.
238 cputype = require(['isainfo', '-k']).decode(default_encoding)
239 # sparc cpus have sun as a target vendor
240 if 'sparc' in cputype:
241 ostype = 'sun-solaris'
242 elif ostype.startswith('MINGW'):
243 # msys' `uname` does not print gcc configuration, but prints msys
244 # configuration. so we cannot believe `uname -m`:
245 # msys1 is always i686 and msys2 is always x86_64.
246 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
248 ostype = 'pc-windows-gnu'
250 if os.environ.get('MSYSTEM') == 'MINGW64':
252 elif ostype.startswith('MSYS'):
253 ostype = 'pc-windows-gnu'
254 elif ostype.startswith('CYGWIN_NT'):
256 if ostype.endswith('WOW64'):
258 ostype = 'pc-windows-gnu'
259 elif sys.platform == 'win32':
260 # Some Windows platforms might have a `uname` command that returns a
261 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
262 # these cases, fall back to using sys.platform.
263 return 'x86_64-pc-windows-msvc'
265 err = "unknown OS type: {}".format(ostype)
268 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
269 cputype = subprocess.check_output(
270 ['uname', '-p']).strip().decode(default_encoding)
273 'aarch64': 'aarch64',
280 'powerpc': 'powerpc',
281 'powerpc64': 'powerpc64',
282 'powerpc64le': 'powerpc64le',
284 'ppc64': 'powerpc64',
285 'ppc64le': 'powerpc64le',
286 'riscv64': 'riscv64gc',
294 # Consider the direct transformation first and then the special cases
295 if cputype in cputype_mapper:
296 cputype = cputype_mapper[cputype]
297 elif cputype in {'xscale', 'arm'}:
299 if ostype == 'linux-android':
300 ostype = 'linux-androideabi'
301 elif ostype == 'unknown-freebsd':
302 cputype = subprocess.check_output(
303 ['uname', '-p']).strip().decode(default_encoding)
304 ostype = 'unknown-freebsd'
305 elif cputype == 'armv6l':
307 if ostype == 'linux-android':
308 ostype = 'linux-androideabi'
311 elif cputype in {'armv7l', 'armv8l'}:
313 if ostype == 'linux-android':
314 ostype = 'linux-androideabi'
317 elif cputype == 'mips':
318 if sys.byteorder == 'big':
320 elif sys.byteorder == 'little':
323 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
324 elif cputype == 'mips64':
325 if sys.byteorder == 'big':
327 elif sys.byteorder == 'little':
330 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
331 # only the n64 ABI is supported, indicate it
333 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
336 err = "unknown cpu type: {}".format(cputype)
339 return "{}-{}".format(cputype, ostype)
342 @contextlib.contextmanager
343 def output(filepath):
344 tmp = filepath + '.tmp'
345 with open(tmp, 'w') as f:
348 if os.path.exists(filepath):
349 os.remove(filepath) # PermissionError/OSError on Win32 if in use
351 shutil.copy2(tmp, filepath)
354 os.rename(tmp, filepath)
357 class Stage0Toolchain:
358 def __init__(self, stage0_payload):
359 self.date = stage0_payload["date"]
360 self.version = stage0_payload["version"]
363 return self.version + "-" + self.date
366 class RustBuild(object):
367 """Provide all the methods required to build Rust"""
369 self.checksums_sha256 = {}
370 self.stage0_compiler = None
371 self.stage0_rustfmt = None
372 self._download_url = ''
376 self.config_toml = ''
378 self.use_locked_deps = ''
379 self.use_vendored_sources = ''
381 self.git_version = None
382 self.nix_deps_dir = None
383 self.rustc_commit = None
385 def download_toolchain(self, stage0=True, rustc_channel=None):
386 """Fetch the build system for Rust, written in Rust
388 This method will build a cache directory, then it will fetch the
389 tarball which has the stage0 compiler used to then bootstrap the Rust
392 Each downloaded tarball is extracted, after that, the script
393 will move all the content to the right place.
395 if rustc_channel is None:
396 rustc_channel = self.stage0_compiler.version
397 bin_root = self.bin_root(stage0)
399 key = self.stage0_compiler.date
401 key += str(self.rustc_commit)
402 if self.rustc(stage0).startswith(bin_root) and \
403 (not os.path.exists(self.rustc(stage0)) or
404 self.program_out_of_date(self.rustc_stamp(stage0), key)):
405 if os.path.exists(bin_root):
406 shutil.rmtree(bin_root)
407 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
408 filename = "rust-std-{}-{}{}".format(
409 rustc_channel, self.build, tarball_suffix)
410 pattern = "rust-std-{}".format(self.build)
411 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
412 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
414 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
415 # download-rustc doesn't need its own cargo, it can just use beta's.
417 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
419 self._download_component_helper(filename, "cargo", tarball_suffix)
420 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
422 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
423 self._download_component_helper(
424 filename, "rustc-dev", tarball_suffix, stage0
427 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
428 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
429 lib_dir = "{}/lib".format(bin_root)
430 for lib in os.listdir(lib_dir):
431 if lib.endswith(".so"):
432 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
433 with output(self.rustc_stamp(stage0)) as rust_stamp:
434 rust_stamp.write(key)
436 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
437 not os.path.exists(self.rustfmt())
438 or self.program_out_of_date(
439 self.rustfmt_stamp(),
440 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
443 if self.stage0_rustfmt is not None:
444 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
445 filename = "rustfmt-{}-{}{}".format(
446 self.stage0_rustfmt.version, self.build, tarball_suffix,
448 self._download_component_helper(
449 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
451 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
452 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
453 with output(self.rustfmt_stamp()) as rustfmt_stamp:
454 rustfmt_stamp.write(self.stage0_rustfmt.channel())
456 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
457 if self.downloading_llvm() and stage0:
458 # We want the most recent LLVM submodule update to avoid downloading
459 # LLVM more often than necessary.
461 # This git command finds that commit SHA, looking for bors-authored
462 # merges that modified src/llvm-project or other relevant version
465 # This works even in a repository that has not yet initialized
467 top_level = subprocess.check_output([
468 "git", "rev-parse", "--show-toplevel",
469 ]).decode(sys.getdefaultencoding()).strip()
470 llvm_sha = subprocess.check_output([
471 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
472 "--merges", "--first-parent", "HEAD",
474 "{}/src/llvm-project".format(top_level),
475 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
476 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
477 "{}/src/version".format(top_level)
478 ]).decode(sys.getdefaultencoding()).strip()
479 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
480 llvm_root = self.llvm_root()
481 llvm_lib = os.path.join(llvm_root, "lib")
482 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
483 self._download_ci_llvm(llvm_sha, llvm_assertions)
484 for binary in ["llvm-config", "FileCheck"]:
485 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
486 for lib in os.listdir(llvm_lib):
487 if lib.endswith(".so"):
488 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
489 with output(self.llvm_stamp()) as llvm_stamp:
490 llvm_stamp.write(llvm_sha + str(llvm_assertions))
492 def downloading_llvm(self):
493 opt = self.get_toml('download-ci-llvm', 'llvm')
494 # This is currently all tier 1 targets (since others may not have CI
496 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
497 supported_platforms = [
498 "aarch64-unknown-linux-gnu",
499 "i686-pc-windows-gnu",
500 "i686-pc-windows-msvc",
501 "i686-unknown-linux-gnu",
502 "x86_64-unknown-linux-gnu",
503 "x86_64-apple-darwin",
504 "x86_64-pc-windows-gnu",
505 "x86_64-pc-windows-msvc",
507 return opt == "true" \
508 or (opt == "if-available" and self.build in supported_platforms)
510 def _download_component_helper(
511 self, filename, pattern, tarball_suffix, stage0=True, key=None
515 key = self.stage0_compiler.date
517 key = self.rustc_commit
518 cache_dst = os.path.join(self.build_dir, "cache")
519 rustc_cache = os.path.join(cache_dst, key)
520 if not os.path.exists(rustc_cache):
521 os.makedirs(rustc_cache)
524 base = self._download_url
525 url = "dist/{}".format(key)
527 base = "https://ci-artifacts.rust-lang.org"
528 url = "rustc-builds/{}".format(self.rustc_commit)
529 tarball = os.path.join(rustc_cache, filename)
530 if not os.path.exists(tarball):
533 "{}/{}".format(url, filename),
535 self.checksums_sha256,
536 verbose=self.verbose,
539 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
541 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
542 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
543 cache_dst = os.path.join(self.build_dir, "cache")
544 rustc_cache = os.path.join(cache_dst, cache_prefix)
545 if not os.path.exists(rustc_cache):
546 os.makedirs(rustc_cache)
548 base = "https://ci-artifacts.rust-lang.org"
549 url = "rustc-builds/{}".format(llvm_sha)
551 url = url.replace('rustc-builds', 'rustc-builds-alt')
552 # ci-artifacts are only stored as .xz, not .gz
554 print("error: XZ support is required to download LLVM")
555 print("help: consider disabling `download-ci-llvm` or using python3")
557 tarball_suffix = '.tar.xz'
558 filename = "rust-dev-nightly-" + self.build + tarball_suffix
559 tarball = os.path.join(rustc_cache, filename)
560 if not os.path.exists(tarball):
563 "{}/{}".format(url, filename),
565 self.checksums_sha256,
566 verbose=self.verbose,
569 unpack(tarball, tarball_suffix, self.llvm_root(),
571 verbose=self.verbose)
573 def fix_bin_or_dylib(self, fname):
574 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
575 or the RPATH section, to fix the dynamic library search path
577 This method is only required on NixOS and uses the PatchELF utility to
578 change the interpreter/RPATH of ELF executables.
580 Please see https://nixos.org/patchelf.html for more information
582 default_encoding = sys.getdefaultencoding()
584 ostype = subprocess.check_output(
585 ['uname', '-s']).strip().decode(default_encoding)
586 except subprocess.CalledProcessError:
588 except OSError as reason:
589 if getattr(reason, 'winerror', None) is not None:
593 if ostype != "Linux":
596 # Use `/etc/os-release` instead of `/etc/NIXOS`.
597 # The latter one does not exist on NixOS when using tmpfs as root.
599 with open("/etc/os-release", "r") as f:
600 if not any(line.strip() == "ID=nixos" for line in f):
602 except FileNotFoundError:
604 if os.path.exists("/lib"):
607 # At this point we're pretty sure the user is running NixOS
608 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
609 print(nix_os_msg, fname)
611 # Only build `.nix-deps` once.
612 nix_deps_dir = self.nix_deps_dir
614 # Run `nix-build` to "build" each dependency (which will likely reuse
615 # the existing `/nix/store` copy, or at most download a pre-built copy).
617 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
618 # directory, but still reference the actual `/nix/store` path in the rpath
619 # as it makes it significantly more robust against changes to the location of
620 # the `.nix-deps` location.
622 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
623 # zlib: Needed as a system dependency of `libLLVM-*.so`.
624 # patchelf: Needed for patching ELF binaries (see doc comment above).
625 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
627 with (import <nixpkgs> {});
629 name = "rust-stage0-dependencies";
638 subprocess.check_output([
639 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
641 except subprocess.CalledProcessError as reason:
642 print("warning: failed to call nix-build:", reason)
644 self.nix_deps_dir = nix_deps_dir
646 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
648 # Relative default, all binary and dynamic libraries we ship
649 # appear to have this (even when `../lib` is redundant).
651 os.path.join(os.path.realpath(nix_deps_dir), "lib")
653 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
654 if not fname.endswith(".so"):
655 # Finally, set the corret .interp for binaries
656 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
657 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
660 subprocess.check_output([patchelf] + patchelf_args + [fname])
661 except subprocess.CalledProcessError as reason:
662 print("warning: failed to call patchelf:", reason)
665 # If `download-rustc` is set, download the most recent commit with CI artifacts
666 def maybe_download_ci_toolchain(self):
667 # If `download-rustc` is not set, default to rebuilding.
668 download_rustc = self.get_toml("download-rustc", section="rust")
669 if download_rustc is None or download_rustc == "false":
671 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
673 # Handle running from a directory other than the top level
674 rev_parse = ["git", "rev-parse", "--show-toplevel"]
675 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
676 compiler = "{}/compiler/".format(top_level)
677 library = "{}/library/".format(top_level)
679 # Look for a version to compare to based on the current commit.
680 # Only commits merged by bors will have CI artifacts.
682 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
683 "--merges", "--first-parent", "HEAD"
685 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
687 # Warn if there were changes to the compiler or standard library since the ancestor commit.
688 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
690 if download_rustc == "if-unchanged":
692 print("warning: `download-rustc` is enabled, but there are changes to \
693 compiler/ or library/")
696 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
697 self.rustc_commit = commit
698 # FIXME: support downloading artifacts from the beta channel
699 self.download_toolchain(False, "nightly")
701 def rustc_stamp(self, stage0):
702 """Return the path for .rustc-stamp at the given stage
705 >>> rb.build_dir = "build"
706 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
708 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
711 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
713 def rustfmt_stamp(self):
714 """Return the path for .rustfmt-stamp
717 >>> rb.build_dir = "build"
718 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
721 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
723 def llvm_stamp(self):
724 """Return the path for .rustfmt-stamp
727 >>> rb.build_dir = "build"
728 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
731 return os.path.join(self.llvm_root(), '.llvm-stamp')
734 def program_out_of_date(self, stamp_path, key):
735 """Check if the given program stamp is out of date"""
736 if not os.path.exists(stamp_path) or self.clean:
738 with open(stamp_path, 'r') as stamp:
739 return key != stamp.read()
741 def bin_root(self, stage0):
742 """Return the binary root directory for the given stage
745 >>> rb.build_dir = "build"
746 >>> rb.bin_root(True) == os.path.join("build", "stage0")
748 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
751 When the 'build' property is given should be a nested directory:
753 >>> rb.build = "devel"
754 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
761 return os.path.join(self.build_dir, self.build, subdir)
764 """Return the CI LLVM root directory
767 >>> rb.build_dir = "build"
768 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
771 When the 'build' property is given should be a nested directory:
773 >>> rb.build = "devel"
774 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
777 return os.path.join(self.build_dir, self.build, "ci-llvm")
779 def get_toml(self, key, section=None):
780 """Returns the value of the given key in config.toml, otherwise returns None
783 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
784 >>> rb.get_toml("key2")
787 If the key does not exists, the result is None:
789 >>> rb.get_toml("key3") is None
792 Optionally also matches the section the key appears in
794 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
795 >>> rb.get_toml('key', 'a')
797 >>> rb.get_toml('key', 'b')
799 >>> rb.get_toml('key', 'c') is None
802 >>> rb.config_toml = 'key1 = true'
803 >>> rb.get_toml("key1")
808 for line in self.config_toml.splitlines():
809 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
810 if section_match is not None:
811 cur_section = section_match.group(1)
813 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
814 if match is not None:
815 value = match.group(1)
816 if section is None or section == cur_section:
817 return self.get_string(value) or value.strip()
821 """Return config path for cargo"""
822 return self.program_config('cargo')
824 def rustc(self, stage0):
825 """Return config path for rustc"""
826 return self.program_config('rustc', stage0)
829 """Return config path for rustfmt"""
830 if self.stage0_rustfmt is None:
832 return self.program_config('rustfmt')
834 def program_config(self, program, stage0=True):
835 """Return config path for the given program at the given stage
838 >>> rb.config_toml = 'rustc = "rustc"\\n'
839 >>> rb.program_config('rustc')
841 >>> rb.config_toml = ''
842 >>> cargo_path = rb.program_config('cargo', True)
843 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
846 >>> cargo_path = rb.program_config('cargo', False)
847 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
851 config = self.get_toml(program)
853 return os.path.expanduser(config)
854 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
855 program, self.exe_suffix()))
858 def get_string(line):
859 """Return the value between double quotes
861 >>> RustBuild.get_string(' "devel" ')
863 >>> RustBuild.get_string(" 'devel' ")
865 >>> RustBuild.get_string('devel') is None
867 >>> RustBuild.get_string(' "devel ')
870 start = line.find('"')
872 end = start + 1 + line[start + 1:].find('"')
873 return line[start + 1:end]
874 start = line.find('\'')
876 end = start + 1 + line[start + 1:].find('\'')
877 return line[start + 1:end]
882 """Return a suffix for executables"""
883 if sys.platform == 'win32':
887 def bootstrap_binary(self):
888 """Return the path of the bootstrap binary
891 >>> rb.build_dir = "build"
892 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
893 ... "debug", "bootstrap")
896 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
898 def build_bootstrap(self):
899 """Build bootstrap"""
900 build_dir = os.path.join(self.build_dir, "bootstrap")
901 if self.clean and os.path.exists(build_dir):
902 shutil.rmtree(build_dir)
903 env = os.environ.copy()
904 # `CARGO_BUILD_TARGET` breaks bootstrap build.
905 # See also: <https://github.com/rust-lang/rust/issues/70208>.
906 if "CARGO_BUILD_TARGET" in env:
907 del env["CARGO_BUILD_TARGET"]
908 env["CARGO_TARGET_DIR"] = build_dir
909 env["RUSTC"] = self.rustc(True)
910 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
911 (os.pathsep + env["LD_LIBRARY_PATH"]) \
912 if "LD_LIBRARY_PATH" in env else ""
913 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
914 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
915 if "DYLD_LIBRARY_PATH" in env else ""
916 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
917 (os.pathsep + env["LIBRARY_PATH"]) \
918 if "LIBRARY_PATH" in env else ""
919 # preserve existing RUSTFLAGS
920 env.setdefault("RUSTFLAGS", "")
921 env["RUSTFLAGS"] += " -Cdebuginfo=2"
923 build_section = "target.{}".format(self.build)
925 if self.get_toml("crt-static", build_section) == "true":
926 target_features += ["+crt-static"]
927 elif self.get_toml("crt-static", build_section) == "false":
928 target_features += ["-crt-static"]
930 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
931 target_linker = self.get_toml("linker", build_section)
932 if target_linker is not None:
933 env["RUSTFLAGS"] += " -C linker=" + target_linker
934 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
935 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
936 if self.get_toml("deny-warnings", "rust") != "false":
937 env["RUSTFLAGS"] += " -Dwarnings"
939 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
940 os.pathsep + env["PATH"]
941 if not os.path.isfile(self.cargo()):
942 raise Exception("no cargo executable found at `{}`".format(
944 args = [self.cargo(), "build", "--manifest-path",
945 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
946 for _ in range(1, self.verbose):
947 args.append("--verbose")
948 if self.use_locked_deps:
949 args.append("--locked")
950 if self.use_vendored_sources:
951 args.append("--frozen")
952 run(args, env=env, verbose=self.verbose)
954 def build_triple(self):
955 """Build triple as in LLVM
957 Note that `default_build_triple` is moderately expensive,
958 so use `self.build` where possible.
960 config = self.get_toml('build')
963 return default_build_triple(self.verbose)
965 def check_submodule(self, module, slow_submodules):
966 if not slow_submodules:
967 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
968 cwd=os.path.join(self.rust_root, module),
969 stdout=subprocess.PIPE)
974 def update_submodule(self, module, checked_out, recorded_submodules):
975 module_path = os.path.join(self.rust_root, module)
977 if checked_out is not None:
978 default_encoding = sys.getdefaultencoding()
979 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
980 if recorded_submodules[module] == checked_out:
983 print("Updating submodule", module)
985 run(["git", "submodule", "-q", "sync", module],
986 cwd=self.rust_root, verbose=self.verbose)
988 update_args = ["git", "submodule", "update", "--init", "--recursive"]
989 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
990 update_args.append("--progress")
991 update_args.append(module)
992 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
994 run(["git", "reset", "-q", "--hard"],
995 cwd=module_path, verbose=self.verbose)
996 run(["git", "clean", "-qdfx"],
997 cwd=module_path, verbose=self.verbose)
999 def update_submodules(self):
1000 """Update submodules"""
1001 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1002 self.get_toml('submodules') == "false":
1005 default_encoding = sys.getdefaultencoding()
1007 # check the existence and version of 'git' command
1008 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1009 self.git_version = distutils.version.LooseVersion(git_version_str)
1011 slow_submodules = self.get_toml('fast-submodules') == "false"
1014 print('Unconditionally updating submodules')
1016 print('Updating only changed submodules')
1017 default_encoding = sys.getdefaultencoding()
1018 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1019 # currently requires everything in a workspace to be "locally present" when starting a
1020 # build, and will give a hard error if any Cargo.toml files are missing.
1021 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1022 # share a workspace with any tools - maybe it could be excluded from the workspace?
1023 # That will still require cloning the submodules the second you check the standard
1024 # library, though...
1025 # FIXME: Is there a way to avoid hard-coding the submodules required?
1026 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1028 "src/tools/rust-installer",
1032 "library/backtrace",
1035 filtered_submodules = []
1036 submodules_names = []
1037 for module in submodules:
1038 check = self.check_submodule(module, slow_submodules)
1039 filtered_submodules.append((module, check))
1040 submodules_names.append(module)
1041 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1042 cwd=self.rust_root, stdout=subprocess.PIPE)
1043 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1044 # { filename: hash }
1045 recorded_submodules = {}
1046 for data in recorded:
1047 # [mode, kind, hash, filename]
1049 recorded_submodules[data[3]] = data[2]
1050 for module in filtered_submodules:
1051 self.update_submodule(module[0], module[1], recorded_submodules)
1052 print("Submodules updated in %.2f seconds" % (time() - start_time))
1054 def set_dist_environment(self, url):
1055 """Set download URL for normal environment"""
1056 if 'RUSTUP_DIST_SERVER' in os.environ:
1057 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1059 self._download_url = url
1061 def check_vendored_status(self):
1062 """Check that vendoring is configured properly"""
1063 vendor_dir = os.path.join(self.rust_root, 'vendor')
1064 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1065 if os.environ.get('USER') != os.environ['SUDO_USER']:
1066 self.use_vendored_sources = True
1067 print('info: looks like you are running this command under `sudo`')
1068 print(' and so in order to preserve your $HOME this will now')
1069 print(' use vendored sources by default.')
1070 if not os.path.exists(vendor_dir):
1071 print('error: vendoring required, but vendor directory does not exist.')
1072 print(' Run `cargo vendor` without sudo to initialize the '
1073 'vendor directory.')
1074 raise Exception("{} not found".format(vendor_dir))
1076 if self.use_vendored_sources:
1077 if not os.path.exists('.cargo'):
1078 os.makedirs('.cargo')
1079 with output('.cargo/config') as cargo_config:
1081 "[source.crates-io]\n"
1082 "replace-with = 'vendored-sources'\n"
1083 "registry = 'https://example.com'\n"
1085 "[source.vendored-sources]\n"
1086 "directory = '{}/vendor'\n"
1087 .format(self.rust_root))
1089 if os.path.exists('.cargo'):
1090 shutil.rmtree('.cargo')
1092 def ensure_vendored(self):
1093 """Ensure that the vendored sources are available if needed"""
1094 vendor_dir = os.path.join(self.rust_root, 'vendor')
1095 # Note that this does not handle updating the vendored dependencies if
1096 # the rust git repository is updated. Normal development usually does
1097 # not use vendoring, so hopefully this isn't too much of a problem.
1098 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1102 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1103 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1104 ], verbose=self.verbose, cwd=self.rust_root)
1107 def bootstrap(help_triggered):
1108 """Configure, fetch, build and run the initial bootstrap"""
1110 # If the user is asking for help, let them know that the whole download-and-build
1111 # process has to happen before anything is printed out.
1113 print("info: Downloading and building bootstrap before processing --help")
1114 print(" command. See src/bootstrap/README.md for help with common")
1117 parser = argparse.ArgumentParser(description='Build rust')
1118 parser.add_argument('--config')
1119 parser.add_argument('--build')
1120 parser.add_argument('--clean', action='store_true')
1121 parser.add_argument('-v', '--verbose', action='count', default=0)
1123 args = [a for a in sys.argv if a != '-h' and a != '--help']
1124 args, _ = parser.parse_known_args(args)
1126 # Configure initial bootstrap
1128 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1129 build.verbose = args.verbose
1130 build.clean = args.clean
1132 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1134 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1135 if not toml_path and os.path.exists('config.toml'):
1136 toml_path = 'config.toml'
1139 if not os.path.exists(toml_path):
1140 toml_path = os.path.join(build.rust_root, toml_path)
1142 with open(toml_path) as config:
1143 build.config_toml = config.read()
1145 profile = build.get_toml('profile')
1146 if profile is not None:
1147 include_file = 'config.{}.toml'.format(profile)
1148 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1149 include_path = os.path.join(include_dir, include_file)
1150 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1151 # specific key, so appending our defaults at the end allows the user to override them
1152 with open(include_path) as included_toml:
1153 build.config_toml += os.linesep + included_toml.read()
1155 config_verbose = build.get_toml('verbose', 'build')
1156 if config_verbose is not None:
1157 build.verbose = max(build.verbose, int(config_verbose))
1159 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1161 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1163 build.check_vendored_status()
1165 build_dir = build.get_toml('build-dir', 'build') or 'build'
1166 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1168 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1170 build.checksums_sha256 = data["checksums_sha256"]
1171 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1172 if data.get("rustfmt") is not None:
1173 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1175 build.set_dist_environment(data["dist_server"])
1177 build.build = args.build or build.build_triple()
1178 build.update_submodules()
1180 # Fetch/build the bootstrap
1181 build.download_toolchain()
1182 # Download the master compiler if `download-rustc` is set
1183 build.maybe_download_ci_toolchain()
1185 build.ensure_vendored()
1186 build.build_bootstrap()
1190 args = [build.bootstrap_binary()]
1191 args.extend(sys.argv[1:])
1192 env = os.environ.copy()
1193 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1194 env["BOOTSTRAP_PYTHON"] = sys.executable
1195 env["BUILD_DIR"] = build.build_dir
1196 env["RUSTC_BOOTSTRAP"] = '1'
1198 env["BOOTSTRAP_CONFIG"] = toml_path
1199 if build.rustc_commit is not None:
1200 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1201 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1205 """Entry point for the bootstrap process"""
1208 # x.py help <cmd> ...
1209 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1210 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1213 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1215 bootstrap(help_triggered)
1216 if not help_triggered:
1217 print("Build completed successfully in {}".format(
1218 format_build_time(time() - start_time)))
1219 except (SystemExit, KeyboardInterrupt) as error:
1220 if hasattr(error, 'code') and isinstance(error.code, int):
1221 exit_code = error.code
1225 if not help_triggered:
1226 print("Build completed unsuccessfully in {}".format(
1227 format_build_time(time() - start_time)))
1231 if __name__ == '__main__':