1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see https://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
154 # For most failures, we definitely do want to print this error, or the user will have no
155 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
156 # have already printed an error above, so there's no need to print the exact command we're
164 def require(cmd, exit=True):
165 '''Run a command, returning its output.
167 If `exit` is `True`, exit the process.
168 Otherwise, return None.'''
170 return subprocess.check_output(cmd).strip()
171 except (subprocess.CalledProcessError, OSError) as exc:
174 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
175 print("Please make sure it's installed and in the path.")
179 def stage0_data(rust_root):
180 """Build a dictionary from stage0.txt"""
181 nightlies = os.path.join(rust_root, "src/stage0.txt")
182 with open(nightlies, 'r') as nightlies:
183 lines = [line.rstrip() for line in nightlies
184 if not line.startswith("#")]
185 return dict([line.split(": ", 1) for line in lines if line])
188 def format_build_time(duration):
189 """Return a nicer format for build time
191 >>> format_build_time('300')
194 return str(datetime.timedelta(seconds=int(duration)))
197 def default_build_triple(verbose):
198 """Build triple as in LLVM"""
199 # If the user already has a host build triple with an existing `rustc`
200 # install, use their preference. This fixes most issues with Windows builds
201 # being detected as GNU instead of MSVC.
202 default_encoding = sys.getdefaultencoding()
204 version = subprocess.check_output(["rustc", "--version", "--verbose"],
205 stderr=subprocess.DEVNULL)
206 version = version.decode(default_encoding)
207 host = next(x for x in version.split('\n') if x.startswith("host: "))
208 triple = host.split("host: ")[1]
210 print("detected default triple {}".format(triple))
212 except Exception as e:
214 print("rustup not detected: {}".format(e))
215 print("falling back to auto-detect")
217 required = sys.platform != 'win32'
218 ostype = require(["uname", "-s"], exit=required)
219 cputype = require(['uname', '-m'], exit=required)
221 # If we do not have `uname`, assume Windows.
222 if ostype is None or cputype is None:
223 return 'x86_64-pc-windows-msvc'
225 ostype = ostype.decode(default_encoding)
226 cputype = cputype.decode(default_encoding)
228 # The goal here is to come up with the same triple as LLVM would,
229 # at least for the subset of platforms we're willing to target.
231 'Darwin': 'apple-darwin',
232 'DragonFly': 'unknown-dragonfly',
233 'FreeBSD': 'unknown-freebsd',
234 'Haiku': 'unknown-haiku',
235 'NetBSD': 'unknown-netbsd',
236 'OpenBSD': 'unknown-openbsd'
239 # Consider the direct transformation first and then the special cases
240 if ostype in ostype_mapper:
241 ostype = ostype_mapper[ostype]
242 elif ostype == 'Linux':
243 os_from_sp = subprocess.check_output(
244 ['uname', '-o']).strip().decode(default_encoding)
245 if os_from_sp == 'Android':
246 ostype = 'linux-android'
248 ostype = 'unknown-linux-gnu'
249 elif ostype == 'SunOS':
250 ostype = 'pc-solaris'
251 # On Solaris, uname -m will return a machine classification instead
252 # of a cpu type, so uname -p is recommended instead. However, the
253 # output from that option is too generic for our purposes (it will
254 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
255 # must be used instead.
256 cputype = require(['isainfo', '-k']).decode(default_encoding)
257 # sparc cpus have sun as a target vendor
258 if 'sparc' in cputype:
259 ostype = 'sun-solaris'
260 elif ostype.startswith('MINGW'):
261 # msys' `uname` does not print gcc configuration, but prints msys
262 # configuration. so we cannot believe `uname -m`:
263 # msys1 is always i686 and msys2 is always x86_64.
264 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
266 ostype = 'pc-windows-gnu'
268 if os.environ.get('MSYSTEM') == 'MINGW64':
270 elif ostype.startswith('MSYS'):
271 ostype = 'pc-windows-gnu'
272 elif ostype.startswith('CYGWIN_NT'):
274 if ostype.endswith('WOW64'):
276 ostype = 'pc-windows-gnu'
277 elif sys.platform == 'win32':
278 # Some Windows platforms might have a `uname` command that returns a
279 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
280 # these cases, fall back to using sys.platform.
281 return 'x86_64-pc-windows-msvc'
283 err = "unknown OS type: {}".format(ostype)
286 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
287 cputype = subprocess.check_output(
288 ['uname', '-p']).strip().decode(default_encoding)
291 'aarch64': 'aarch64',
298 'powerpc': 'powerpc',
299 'powerpc64': 'powerpc64',
300 'powerpc64le': 'powerpc64le',
302 'ppc64': 'powerpc64',
303 'ppc64le': 'powerpc64le',
304 'riscv64': 'riscv64gc',
312 # Consider the direct transformation first and then the special cases
313 if cputype in cputype_mapper:
314 cputype = cputype_mapper[cputype]
315 elif cputype in {'xscale', 'arm'}:
317 if ostype == 'linux-android':
318 ostype = 'linux-androideabi'
319 elif ostype == 'unknown-freebsd':
320 cputype = subprocess.check_output(
321 ['uname', '-p']).strip().decode(default_encoding)
322 ostype = 'unknown-freebsd'
323 elif cputype == 'armv6l':
325 if ostype == 'linux-android':
326 ostype = 'linux-androideabi'
329 elif cputype in {'armv7l', 'armv8l'}:
331 if ostype == 'linux-android':
332 ostype = 'linux-androideabi'
335 elif cputype == 'mips':
336 if sys.byteorder == 'big':
338 elif sys.byteorder == 'little':
341 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
342 elif cputype == 'mips64':
343 if sys.byteorder == 'big':
345 elif sys.byteorder == 'little':
348 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
349 # only the n64 ABI is supported, indicate it
351 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
354 err = "unknown cpu type: {}".format(cputype)
357 return "{}-{}".format(cputype, ostype)
360 @contextlib.contextmanager
361 def output(filepath):
362 tmp = filepath + '.tmp'
363 with open(tmp, 'w') as f:
366 if os.path.exists(filepath):
367 os.remove(filepath) # PermissionError/OSError on Win32 if in use
369 shutil.copy2(tmp, filepath)
372 os.rename(tmp, filepath)
375 class RustBuild(object):
376 """Provide all the methods required to build Rust"""
379 self._download_url = ''
380 self.rustc_channel = ''
381 self.rustfmt_channel = ''
385 self.config_toml = ''
387 self.use_locked_deps = ''
388 self.use_vendored_sources = ''
390 self.git_version = None
391 self.nix_deps_dir = None
392 self.rustc_commit = None
394 def download_toolchain(self, stage0=True, rustc_channel=None):
395 """Fetch the build system for Rust, written in Rust
397 This method will build a cache directory, then it will fetch the
398 tarball which has the stage0 compiler used to then bootstrap the Rust
401 Each downloaded tarball is extracted, after that, the script
402 will move all the content to the right place.
404 if rustc_channel is None:
405 rustc_channel = self.rustc_channel
406 rustfmt_channel = self.rustfmt_channel
407 bin_root = self.bin_root(stage0)
411 key += str(self.rustc_commit)
412 if self.rustc(stage0).startswith(bin_root) and \
413 (not os.path.exists(self.rustc(stage0)) or
414 self.program_out_of_date(self.rustc_stamp(stage0), key)):
415 if os.path.exists(bin_root):
416 shutil.rmtree(bin_root)
417 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
418 filename = "rust-std-{}-{}{}".format(
419 rustc_channel, self.build, tarball_suffix)
420 pattern = "rust-std-{}".format(self.build)
421 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
422 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
424 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
425 # download-rustc doesn't need its own cargo, it can just use beta's.
427 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
429 self._download_component_helper(filename, "cargo", tarball_suffix)
430 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
432 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
433 self._download_component_helper(
434 filename, "rustc-dev", tarball_suffix, stage0
437 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
438 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
439 lib_dir = "{}/lib".format(bin_root)
440 for lib in os.listdir(lib_dir):
441 if lib.endswith(".so"):
442 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
443 with output(self.rustc_stamp(stage0)) as rust_stamp:
444 rust_stamp.write(key)
446 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
447 not os.path.exists(self.rustfmt())
448 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
451 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
452 [channel, date] = rustfmt_channel.split('-', 1)
453 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
454 self._download_component_helper(
455 filename, "rustfmt-preview", tarball_suffix, key=date
457 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
458 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
459 with output(self.rustfmt_stamp()) as rustfmt_stamp:
460 rustfmt_stamp.write(self.rustfmt_channel)
462 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
463 if self.downloading_llvm() and stage0:
464 # We want the most recent LLVM submodule update to avoid downloading
465 # LLVM more often than necessary.
467 # This git command finds that commit SHA, looking for bors-authored
468 # merges that modified src/llvm-project or other relevant version
471 # This works even in a repository that has not yet initialized
473 top_level = subprocess.check_output([
474 "git", "rev-parse", "--show-toplevel",
475 ]).decode(sys.getdefaultencoding()).strip()
476 llvm_sha = subprocess.check_output([
477 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
478 "--merges", "--first-parent", "HEAD",
480 "{}/src/llvm-project".format(top_level),
481 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
482 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
483 "{}/src/version".format(top_level)
484 ]).decode(sys.getdefaultencoding()).strip()
485 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
486 llvm_root = self.llvm_root()
487 llvm_lib = os.path.join(llvm_root, "lib")
488 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
489 self._download_ci_llvm(llvm_sha, llvm_assertions)
490 for binary in ["llvm-config", "FileCheck"]:
491 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
492 for lib in os.listdir(llvm_lib):
493 if lib.endswith(".so"):
494 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
495 with output(self.llvm_stamp()) as llvm_stamp:
496 llvm_stamp.write(llvm_sha + str(llvm_assertions))
498 def downloading_llvm(self):
499 opt = self.get_toml('download-ci-llvm', 'llvm')
500 # This is currently all tier 1 targets (since others may not have CI
502 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
503 supported_platforms = [
504 "aarch64-unknown-linux-gnu",
505 "i686-pc-windows-gnu",
506 "i686-pc-windows-msvc",
507 "i686-unknown-linux-gnu",
508 "x86_64-unknown-linux-gnu",
509 "x86_64-apple-darwin",
510 "x86_64-pc-windows-gnu",
511 "x86_64-pc-windows-msvc",
513 return opt == "true" \
514 or (opt == "if-available" and self.build in supported_platforms)
516 def _download_component_helper(
517 self, filename, pattern, tarball_suffix, stage0=True, key=None
523 key = self.rustc_commit
524 cache_dst = os.path.join(self.build_dir, "cache")
525 rustc_cache = os.path.join(cache_dst, key)
526 if not os.path.exists(rustc_cache):
527 os.makedirs(rustc_cache)
530 url = "{}/dist/{}".format(self._download_url, key)
532 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
533 tarball = os.path.join(rustc_cache, filename)
534 if not os.path.exists(tarball):
535 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
536 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
538 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
539 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
540 cache_dst = os.path.join(self.build_dir, "cache")
541 rustc_cache = os.path.join(cache_dst, cache_prefix)
542 if not os.path.exists(rustc_cache):
543 os.makedirs(rustc_cache)
545 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
547 url = url.replace('rustc-builds', 'rustc-builds-alt')
548 # ci-artifacts are only stored as .xz, not .gz
550 print("error: XZ support is required to download LLVM")
551 print("help: consider disabling `download-ci-llvm` or using python3")
553 tarball_suffix = '.tar.xz'
554 filename = "rust-dev-nightly-" + self.build + tarball_suffix
555 tarball = os.path.join(rustc_cache, filename)
556 if not os.path.exists(tarball):
557 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
558 unpack(tarball, tarball_suffix, self.llvm_root(),
560 verbose=self.verbose)
562 def fix_bin_or_dylib(self, fname):
563 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
564 or the RPATH section, to fix the dynamic library search path
566 This method is only required on NixOS and uses the PatchELF utility to
567 change the interpreter/RPATH of ELF executables.
569 Please see https://nixos.org/patchelf.html for more information
571 default_encoding = sys.getdefaultencoding()
573 ostype = subprocess.check_output(
574 ['uname', '-s']).strip().decode(default_encoding)
575 except subprocess.CalledProcessError:
577 except OSError as reason:
578 if getattr(reason, 'winerror', None) is not None:
582 if ostype != "Linux":
585 # Use `/etc/os-release` instead of `/etc/NIXOS`.
586 # The latter one does not exist on NixOS when using tmpfs as root.
588 with open("/etc/os-release", "r") as f:
589 if not any(line.strip() == "ID=nixos" for line in f):
591 except FileNotFoundError:
593 if os.path.exists("/lib"):
596 # At this point we're pretty sure the user is running NixOS
597 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
598 print(nix_os_msg, fname)
600 # Only build `.nix-deps` once.
601 nix_deps_dir = self.nix_deps_dir
603 # Run `nix-build` to "build" each dependency (which will likely reuse
604 # the existing `/nix/store` copy, or at most download a pre-built copy).
606 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
607 # directory, but still reference the actual `/nix/store` path in the rpath
608 # as it makes it significantly more robust against changes to the location of
609 # the `.nix-deps` location.
611 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
612 # zlib: Needed as a system dependency of `libLLVM-*.so`.
613 # patchelf: Needed for patching ELF binaries (see doc comment above).
614 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
616 with (import <nixpkgs> {});
618 name = "rust-stage0-dependencies";
627 subprocess.check_output([
628 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
630 except subprocess.CalledProcessError as reason:
631 print("warning: failed to call nix-build:", reason)
633 self.nix_deps_dir = nix_deps_dir
635 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
637 # Relative default, all binary and dynamic libraries we ship
638 # appear to have this (even when `../lib` is redundant).
640 os.path.join(os.path.realpath(nix_deps_dir), "lib")
642 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
643 if not fname.endswith(".so"):
644 # Finally, set the corret .interp for binaries
645 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
646 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
649 subprocess.check_output([patchelf] + patchelf_args + [fname])
650 except subprocess.CalledProcessError as reason:
651 print("warning: failed to call patchelf:", reason)
654 # If `download-rustc` is set, download the most recent commit with CI artifacts
655 def maybe_download_ci_toolchain(self):
656 # If `download-rustc` is not set, default to rebuilding.
657 download_rustc = self.get_toml("download-rustc", section="rust")
658 if download_rustc is None or download_rustc == "false":
660 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
662 # Handle running from a directory other than the top level
663 rev_parse = ["git", "rev-parse", "--show-toplevel"]
664 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
665 compiler = "{}/compiler/".format(top_level)
666 library = "{}/library/".format(top_level)
668 # Look for a version to compare to based on the current commit.
669 # Only commits merged by bors will have CI artifacts.
671 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
672 "--merges", "--first-parent", "HEAD"
674 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
676 # Warn if there were changes to the compiler or standard library since the ancestor commit.
677 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
679 if download_rustc == "if-unchanged":
681 print("warning: `download-rustc` is enabled, but there are changes to \
682 compiler/ or library/")
685 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
686 self.rustc_commit = commit
687 # FIXME: support downloading artifacts from the beta channel
688 self.download_toolchain(False, "nightly")
690 def rustc_stamp(self, stage0):
691 """Return the path for .rustc-stamp at the given stage
694 >>> rb.build_dir = "build"
695 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
697 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
700 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
702 def rustfmt_stamp(self):
703 """Return the path for .rustfmt-stamp
706 >>> rb.build_dir = "build"
707 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
710 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
712 def llvm_stamp(self):
713 """Return the path for .rustfmt-stamp
716 >>> rb.build_dir = "build"
717 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
720 return os.path.join(self.llvm_root(), '.llvm-stamp')
723 def program_out_of_date(self, stamp_path, key):
724 """Check if the given program stamp is out of date"""
725 if not os.path.exists(stamp_path) or self.clean:
727 with open(stamp_path, 'r') as stamp:
728 return key != stamp.read()
730 def bin_root(self, stage0):
731 """Return the binary root directory for the given stage
734 >>> rb.build_dir = "build"
735 >>> rb.bin_root(True) == os.path.join("build", "stage0")
737 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
740 When the 'build' property is given should be a nested directory:
742 >>> rb.build = "devel"
743 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
750 return os.path.join(self.build_dir, self.build, subdir)
753 """Return the CI LLVM root directory
756 >>> rb.build_dir = "build"
757 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
760 When the 'build' property is given should be a nested directory:
762 >>> rb.build = "devel"
763 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
766 return os.path.join(self.build_dir, self.build, "ci-llvm")
768 def get_toml(self, key, section=None):
769 """Returns the value of the given key in config.toml, otherwise returns None
772 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
773 >>> rb.get_toml("key2")
776 If the key does not exists, the result is None:
778 >>> rb.get_toml("key3") is None
781 Optionally also matches the section the key appears in
783 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
784 >>> rb.get_toml('key', 'a')
786 >>> rb.get_toml('key', 'b')
788 >>> rb.get_toml('key', 'c') is None
791 >>> rb.config_toml = 'key1 = true'
792 >>> rb.get_toml("key1")
797 for line in self.config_toml.splitlines():
798 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
799 if section_match is not None:
800 cur_section = section_match.group(1)
802 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
803 if match is not None:
804 value = match.group(1)
805 if section is None or section == cur_section:
806 return self.get_string(value) or value.strip()
810 """Return config path for cargo"""
811 return self.program_config('cargo')
813 def rustc(self, stage0):
814 """Return config path for rustc"""
815 return self.program_config('rustc', stage0)
818 """Return config path for rustfmt"""
819 if not self.rustfmt_channel:
821 return self.program_config('rustfmt')
823 def program_config(self, program, stage0=True):
824 """Return config path for the given program at the given stage
827 >>> rb.config_toml = 'rustc = "rustc"\\n'
828 >>> rb.program_config('rustc')
830 >>> rb.config_toml = ''
831 >>> cargo_path = rb.program_config('cargo', True)
832 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
835 >>> cargo_path = rb.program_config('cargo', False)
836 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
840 config = self.get_toml(program)
842 return os.path.expanduser(config)
843 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
844 program, self.exe_suffix()))
847 def get_string(line):
848 """Return the value between double quotes
850 >>> RustBuild.get_string(' "devel" ')
852 >>> RustBuild.get_string(" 'devel' ")
854 >>> RustBuild.get_string('devel') is None
856 >>> RustBuild.get_string(' "devel ')
859 start = line.find('"')
861 end = start + 1 + line[start + 1:].find('"')
862 return line[start + 1:end]
863 start = line.find('\'')
865 end = start + 1 + line[start + 1:].find('\'')
866 return line[start + 1:end]
871 """Return a suffix for executables"""
872 if sys.platform == 'win32':
876 def bootstrap_binary(self):
877 """Return the path of the bootstrap binary
880 >>> rb.build_dir = "build"
881 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
882 ... "debug", "bootstrap")
885 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
887 def build_bootstrap(self):
888 """Build bootstrap"""
889 build_dir = os.path.join(self.build_dir, "bootstrap")
890 if self.clean and os.path.exists(build_dir):
891 shutil.rmtree(build_dir)
892 env = os.environ.copy()
893 # `CARGO_BUILD_TARGET` breaks bootstrap build.
894 # See also: <https://github.com/rust-lang/rust/issues/70208>.
895 if "CARGO_BUILD_TARGET" in env:
896 del env["CARGO_BUILD_TARGET"]
897 env["CARGO_TARGET_DIR"] = build_dir
898 env["RUSTC"] = self.rustc(True)
899 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
900 (os.pathsep + env["LD_LIBRARY_PATH"]) \
901 if "LD_LIBRARY_PATH" in env else ""
902 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
903 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
904 if "DYLD_LIBRARY_PATH" in env else ""
905 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
906 (os.pathsep + env["LIBRARY_PATH"]) \
907 if "LIBRARY_PATH" in env else ""
908 # preserve existing RUSTFLAGS
909 env.setdefault("RUSTFLAGS", "")
910 env["RUSTFLAGS"] += " -Cdebuginfo=2"
912 build_section = "target.{}".format(self.build)
914 if self.get_toml("crt-static", build_section) == "true":
915 target_features += ["+crt-static"]
916 elif self.get_toml("crt-static", build_section) == "false":
917 target_features += ["-crt-static"]
919 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
920 target_linker = self.get_toml("linker", build_section)
921 if target_linker is not None:
922 env["RUSTFLAGS"] += " -C linker=" + target_linker
923 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
924 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
925 if self.get_toml("deny-warnings", "rust") != "false":
926 env["RUSTFLAGS"] += " -Dwarnings"
928 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
929 os.pathsep + env["PATH"]
930 if not os.path.isfile(self.cargo()):
931 raise Exception("no cargo executable found at `{}`".format(
933 args = [self.cargo(), "build", "--manifest-path",
934 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
935 for _ in range(1, self.verbose):
936 args.append("--verbose")
937 if self.use_locked_deps:
938 args.append("--locked")
939 if self.use_vendored_sources:
940 args.append("--frozen")
941 run(args, env=env, verbose=self.verbose)
943 def build_triple(self):
944 """Build triple as in LLVM
946 Note that `default_build_triple` is moderately expensive,
947 so use `self.build` where possible.
949 config = self.get_toml('build')
952 return default_build_triple(self.verbose)
954 def check_submodule(self, module, slow_submodules):
955 if not slow_submodules:
956 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
957 cwd=os.path.join(self.rust_root, module),
958 stdout=subprocess.PIPE)
963 def update_submodule(self, module, checked_out, recorded_submodules):
964 module_path = os.path.join(self.rust_root, module)
966 if checked_out is not None:
967 default_encoding = sys.getdefaultencoding()
968 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
969 if recorded_submodules[module] == checked_out:
972 print("Updating submodule", module)
974 run(["git", "submodule", "-q", "sync", module],
975 cwd=self.rust_root, verbose=self.verbose)
977 update_args = ["git", "submodule", "update", "--init", "--recursive"]
978 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
979 update_args.append("--progress")
980 update_args.append(module)
981 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
983 run(["git", "reset", "-q", "--hard"],
984 cwd=module_path, verbose=self.verbose)
985 run(["git", "clean", "-qdfx"],
986 cwd=module_path, verbose=self.verbose)
988 def update_submodules(self):
989 """Update submodules"""
990 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
991 self.get_toml('submodules') == "false":
994 default_encoding = sys.getdefaultencoding()
996 # check the existence and version of 'git' command
997 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
998 self.git_version = distutils.version.LooseVersion(git_version_str)
1000 slow_submodules = self.get_toml('fast-submodules') == "false"
1003 print('Unconditionally updating submodules')
1005 print('Updating only changed submodules')
1006 default_encoding = sys.getdefaultencoding()
1007 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1008 # currently requires everything in a workspace to be "locally present" when starting a
1009 # build, and will give a hard error if any Cargo.toml files are missing.
1010 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1011 # share a workspace with any tools - maybe it could be excluded from the workspace?
1012 # That will still require cloning the submodules the second you check the standard
1013 # library, though...
1014 # FIXME: Is there a way to avoid hard-coding the submodules required?
1015 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1017 "src/tools/rust-installer",
1021 "library/backtrace",
1024 filtered_submodules = []
1025 submodules_names = []
1026 for module in submodules:
1027 check = self.check_submodule(module, slow_submodules)
1028 filtered_submodules.append((module, check))
1029 submodules_names.append(module)
1030 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1031 cwd=self.rust_root, stdout=subprocess.PIPE)
1032 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1033 # { filename: hash }
1034 recorded_submodules = {}
1035 for data in recorded:
1036 # [mode, kind, hash, filename]
1038 recorded_submodules[data[3]] = data[2]
1039 for module in filtered_submodules:
1040 self.update_submodule(module[0], module[1], recorded_submodules)
1041 print("Submodules updated in %.2f seconds" % (time() - start_time))
1043 def set_normal_environment(self):
1044 """Set download URL for normal environment"""
1045 if 'RUSTUP_DIST_SERVER' in os.environ:
1046 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1048 self._download_url = 'https://static.rust-lang.org'
1050 def set_dev_environment(self):
1051 """Set download URL for development environment"""
1052 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1053 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1055 self._download_url = 'https://dev-static.rust-lang.org'
1057 def check_vendored_status(self):
1058 """Check that vendoring is configured properly"""
1059 vendor_dir = os.path.join(self.rust_root, 'vendor')
1060 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1061 if os.environ.get('USER') != os.environ['SUDO_USER']:
1062 self.use_vendored_sources = True
1063 print('info: looks like you are running this command under `sudo`')
1064 print(' and so in order to preserve your $HOME this will now')
1065 print(' use vendored sources by default.')
1066 if not os.path.exists(vendor_dir):
1067 print('error: vendoring required, but vendor directory does not exist.')
1068 print(' Run `cargo vendor` without sudo to initialize the '
1069 'vendor directory.')
1070 raise Exception("{} not found".format(vendor_dir))
1072 if self.use_vendored_sources:
1073 if not os.path.exists('.cargo'):
1074 os.makedirs('.cargo')
1075 with output('.cargo/config') as cargo_config:
1077 "[source.crates-io]\n"
1078 "replace-with = 'vendored-sources'\n"
1079 "registry = 'https://example.com'\n"
1081 "[source.vendored-sources]\n"
1082 "directory = '{}/vendor'\n"
1083 .format(self.rust_root))
1085 if os.path.exists('.cargo'):
1086 shutil.rmtree('.cargo')
1088 def ensure_vendored(self):
1089 """Ensure that the vendored sources are available if needed"""
1090 vendor_dir = os.path.join(self.rust_root, 'vendor')
1091 # Note that this does not handle updating the vendored dependencies if
1092 # the rust git repository is updated. Normal development usually does
1093 # not use vendoring, so hopefully this isn't too much of a problem.
1094 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1098 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1099 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1100 ], verbose=self.verbose, cwd=self.rust_root)
1103 def bootstrap(help_triggered):
1104 """Configure, fetch, build and run the initial bootstrap"""
1106 # If the user is asking for help, let them know that the whole download-and-build
1107 # process has to happen before anything is printed out.
1109 print("info: Downloading and building bootstrap before processing --help")
1110 print(" command. See src/bootstrap/README.md for help with common")
1113 parser = argparse.ArgumentParser(description='Build rust')
1114 parser.add_argument('--config')
1115 parser.add_argument('--build')
1116 parser.add_argument('--clean', action='store_true')
1117 parser.add_argument('-v', '--verbose', action='count', default=0)
1119 args = [a for a in sys.argv if a != '-h' and a != '--help']
1120 args, _ = parser.parse_known_args(args)
1122 # Configure initial bootstrap
1124 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1125 build.verbose = args.verbose
1126 build.clean = args.clean
1128 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1130 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1131 if not toml_path and os.path.exists('config.toml'):
1132 toml_path = 'config.toml'
1135 if not os.path.exists(toml_path):
1136 toml_path = os.path.join(build.rust_root, toml_path)
1138 with open(toml_path) as config:
1139 build.config_toml = config.read()
1141 profile = build.get_toml('profile')
1142 if profile is not None:
1143 include_file = 'config.{}.toml'.format(profile)
1144 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1145 include_path = os.path.join(include_dir, include_file)
1146 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1147 # specific key, so appending our defaults at the end allows the user to override them
1148 with open(include_path) as included_toml:
1149 build.config_toml += os.linesep + included_toml.read()
1151 config_verbose = build.get_toml('verbose', 'build')
1152 if config_verbose is not None:
1153 build.verbose = max(build.verbose, int(config_verbose))
1155 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1157 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1159 build.check_vendored_status()
1161 build_dir = build.get_toml('build-dir', 'build') or 'build'
1162 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1164 data = stage0_data(build.rust_root)
1165 build.date = data['date']
1166 build.rustc_channel = data['rustc']
1168 if "rustfmt" in data:
1169 build.rustfmt_channel = data['rustfmt']
1172 build.set_dev_environment()
1174 build.set_normal_environment()
1176 build.build = args.build or build.build_triple()
1177 build.update_submodules()
1179 # Fetch/build the bootstrap
1180 build.download_toolchain()
1181 # Download the master compiler if `download-rustc` is set
1182 build.maybe_download_ci_toolchain()
1184 build.ensure_vendored()
1185 build.build_bootstrap()
1189 args = [build.bootstrap_binary()]
1190 args.extend(sys.argv[1:])
1191 env = os.environ.copy()
1192 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1193 env["BOOTSTRAP_PYTHON"] = sys.executable
1194 env["BUILD_DIR"] = build.build_dir
1195 env["RUSTC_BOOTSTRAP"] = '1'
1197 env["BOOTSTRAP_CONFIG"] = toml_path
1198 if build.rustc_commit is not None:
1199 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1200 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1204 """Entry point for the bootstrap process"""
1207 # x.py help <cmd> ...
1208 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1209 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1212 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1214 bootstrap(help_triggered)
1215 if not help_triggered:
1216 print("Build completed successfully in {}".format(
1217 format_build_time(time() - start_time)))
1218 except (SystemExit, KeyboardInterrupt) as error:
1219 if hasattr(error, 'code') and isinstance(error.code, int):
1220 exit_code = error.code
1224 if not help_triggered:
1225 print("Build completed unsuccessfully in {}".format(
1226 format_build_time(time() - start_time)))
1230 if __name__ == '__main__':