1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"],
198 stderr=subprocess.DEVNULL)
199 version = version.decode(default_encoding)
200 host = next(x for x in version.split('\n') if x.startswith("host: "))
201 triple = host.split("host: ")[1]
203 print("detected default triple {}".format(triple))
205 except Exception as e:
207 print("rustup not detected: {}".format(e))
208 print("falling back to auto-detect")
210 required = sys.platform != 'win32'
211 ostype = require(["uname", "-s"], exit=required)
212 cputype = require(['uname', '-m'], exit=required)
214 # If we do not have `uname`, assume Windows.
215 if ostype is None or cputype is None:
216 return 'x86_64-pc-windows-msvc'
218 ostype = ostype.decode(default_encoding)
219 cputype = cputype.decode(default_encoding)
221 # The goal here is to come up with the same triple as LLVM would,
222 # at least for the subset of platforms we're willing to target.
224 'Darwin': 'apple-darwin',
225 'DragonFly': 'unknown-dragonfly',
226 'FreeBSD': 'unknown-freebsd',
227 'Haiku': 'unknown-haiku',
228 'NetBSD': 'unknown-netbsd',
229 'OpenBSD': 'unknown-openbsd'
232 # Consider the direct transformation first and then the special cases
233 if ostype in ostype_mapper:
234 ostype = ostype_mapper[ostype]
235 elif ostype == 'Linux':
236 os_from_sp = subprocess.check_output(
237 ['uname', '-o']).strip().decode(default_encoding)
238 if os_from_sp == 'Android':
239 ostype = 'linux-android'
241 ostype = 'unknown-linux-gnu'
242 elif ostype == 'SunOS':
243 ostype = 'pc-solaris'
244 # On Solaris, uname -m will return a machine classification instead
245 # of a cpu type, so uname -p is recommended instead. However, the
246 # output from that option is too generic for our purposes (it will
247 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
248 # must be used instead.
249 cputype = require(['isainfo', '-k']).decode(default_encoding)
250 # sparc cpus have sun as a target vendor
251 if 'sparc' in cputype:
252 ostype = 'sun-solaris'
253 elif ostype.startswith('MINGW'):
254 # msys' `uname` does not print gcc configuration, but prints msys
255 # configuration. so we cannot believe `uname -m`:
256 # msys1 is always i686 and msys2 is always x86_64.
257 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
259 ostype = 'pc-windows-gnu'
261 if os.environ.get('MSYSTEM') == 'MINGW64':
263 elif ostype.startswith('MSYS'):
264 ostype = 'pc-windows-gnu'
265 elif ostype.startswith('CYGWIN_NT'):
267 if ostype.endswith('WOW64'):
269 ostype = 'pc-windows-gnu'
270 elif sys.platform == 'win32':
271 # Some Windows platforms might have a `uname` command that returns a
272 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
273 # these cases, fall back to using sys.platform.
274 return 'x86_64-pc-windows-msvc'
276 err = "unknown OS type: {}".format(ostype)
279 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
280 cputype = subprocess.check_output(
281 ['uname', '-p']).strip().decode(default_encoding)
284 'aarch64': 'aarch64',
291 'powerpc': 'powerpc',
292 'powerpc64': 'powerpc64',
293 'powerpc64le': 'powerpc64le',
295 'ppc64': 'powerpc64',
296 'ppc64le': 'powerpc64le',
304 # Consider the direct transformation first and then the special cases
305 if cputype in cputype_mapper:
306 cputype = cputype_mapper[cputype]
307 elif cputype in {'xscale', 'arm'}:
309 if ostype == 'linux-android':
310 ostype = 'linux-androideabi'
311 elif ostype == 'unknown-freebsd':
312 cputype = subprocess.check_output(
313 ['uname', '-p']).strip().decode(default_encoding)
314 ostype = 'unknown-freebsd'
315 elif cputype == 'armv6l':
317 if ostype == 'linux-android':
318 ostype = 'linux-androideabi'
321 elif cputype in {'armv7l', 'armv8l'}:
323 if ostype == 'linux-android':
324 ostype = 'linux-androideabi'
327 elif cputype == 'mips':
328 if sys.byteorder == 'big':
330 elif sys.byteorder == 'little':
333 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
334 elif cputype == 'mips64':
335 if sys.byteorder == 'big':
337 elif sys.byteorder == 'little':
340 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
341 # only the n64 ABI is supported, indicate it
343 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
346 err = "unknown cpu type: {}".format(cputype)
349 return "{}-{}".format(cputype, ostype)
352 @contextlib.contextmanager
353 def output(filepath):
354 tmp = filepath + '.tmp'
355 with open(tmp, 'w') as f:
358 if os.path.exists(filepath):
359 os.remove(filepath) # PermissionError/OSError on Win32 if in use
361 shutil.copy2(tmp, filepath)
364 os.rename(tmp, filepath)
367 class RustBuild(object):
368 """Provide all the methods required to build Rust"""
371 self._download_url = ''
372 self.rustc_channel = ''
373 self.rustfmt_channel = ''
377 self.config_toml = ''
379 self.use_locked_deps = ''
380 self.use_vendored_sources = ''
382 self.git_version = None
383 self.nix_deps_dir = None
384 self.rustc_commit = None
386 def download_toolchain(self, stage0=True, rustc_channel=None):
387 """Fetch the build system for Rust, written in Rust
389 This method will build a cache directory, then it will fetch the
390 tarball which has the stage0 compiler used to then bootstrap the Rust
393 Each downloaded tarball is extracted, after that, the script
394 will move all the content to the right place.
396 if rustc_channel is None:
397 rustc_channel = self.rustc_channel
398 rustfmt_channel = self.rustfmt_channel
399 bin_root = self.bin_root(stage0)
403 key += str(self.rustc_commit)
404 if self.rustc(stage0).startswith(bin_root) and \
405 (not os.path.exists(self.rustc(stage0)) or
406 self.program_out_of_date(self.rustc_stamp(stage0), key)):
407 if os.path.exists(bin_root):
408 shutil.rmtree(bin_root)
409 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
410 filename = "rust-std-{}-{}{}".format(
411 rustc_channel, self.build, tarball_suffix)
412 pattern = "rust-std-{}".format(self.build)
413 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
414 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
416 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
417 # download-rustc doesn't need its own cargo, it can just use beta's.
419 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
421 self._download_component_helper(filename, "cargo", tarball_suffix)
422 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
424 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
425 self._download_component_helper(
426 filename, "rustc-dev", tarball_suffix, stage0
429 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
430 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
431 lib_dir = "{}/lib".format(bin_root)
432 for lib in os.listdir(lib_dir):
433 if lib.endswith(".so"):
434 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
435 with output(self.rustc_stamp(stage0)) as rust_stamp:
436 rust_stamp.write(key)
438 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
439 not os.path.exists(self.rustfmt())
440 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
443 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
444 [channel, date] = rustfmt_channel.split('-', 1)
445 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
446 self._download_component_helper(
447 filename, "rustfmt-preview", tarball_suffix, key=date
449 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
450 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
451 with output(self.rustfmt_stamp()) as rustfmt_stamp:
452 rustfmt_stamp.write(self.rustfmt_channel)
454 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
455 if self.downloading_llvm() and stage0:
456 # We want the most recent LLVM submodule update to avoid downloading
457 # LLVM more often than necessary.
459 # This git command finds that commit SHA, looking for bors-authored
460 # merges that modified src/llvm-project.
462 # This works even in a repository that has not yet initialized
464 top_level = subprocess.check_output([
465 "git", "rev-parse", "--show-toplevel",
466 ]).decode(sys.getdefaultencoding()).strip()
467 llvm_sha = subprocess.check_output([
468 "git", "log", "--author=bors", "--format=%H", "-n1",
469 "-m", "--first-parent",
471 "{}/src/llvm-project".format(top_level),
472 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
473 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
474 "{}/src/version".format(top_level)
475 ]).decode(sys.getdefaultencoding()).strip()
476 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
477 llvm_root = self.llvm_root()
478 llvm_lib = os.path.join(llvm_root, "lib")
479 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
480 self._download_ci_llvm(llvm_sha, llvm_assertions)
481 for binary in ["llvm-config", "FileCheck"]:
482 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
483 for lib in os.listdir(llvm_lib):
484 if lib.endswith(".so"):
485 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
486 with output(self.llvm_stamp()) as llvm_stamp:
487 llvm_stamp.write(llvm_sha + str(llvm_assertions))
489 def downloading_llvm(self):
490 opt = self.get_toml('download-ci-llvm', 'llvm')
491 # This is currently all tier 1 targets (since others may not have CI
493 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
494 supported_platforms = [
495 "aarch64-unknown-linux-gnu",
496 "i686-pc-windows-gnu",
497 "i686-pc-windows-msvc",
498 "i686-unknown-linux-gnu",
499 "x86_64-unknown-linux-gnu",
500 "x86_64-apple-darwin",
501 "x86_64-pc-windows-gnu",
502 "x86_64-pc-windows-msvc",
504 return opt == "true" \
505 or (opt == "if-available" and self.build in supported_platforms)
507 def _download_component_helper(
508 self, filename, pattern, tarball_suffix, stage0=True, key=None
514 key = self.rustc_commit
515 cache_dst = os.path.join(self.build_dir, "cache")
516 rustc_cache = os.path.join(cache_dst, key)
517 if not os.path.exists(rustc_cache):
518 os.makedirs(rustc_cache)
521 url = "{}/dist/{}".format(self._download_url, key)
523 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
524 tarball = os.path.join(rustc_cache, filename)
525 if not os.path.exists(tarball):
526 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
527 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
529 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
530 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
531 cache_dst = os.path.join(self.build_dir, "cache")
532 rustc_cache = os.path.join(cache_dst, cache_prefix)
533 if not os.path.exists(rustc_cache):
534 os.makedirs(rustc_cache)
536 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
538 url = url.replace('rustc-builds', 'rustc-builds-alt')
539 # ci-artifacts are only stored as .xz, not .gz
541 print("error: XZ support is required to download LLVM")
542 print("help: consider disabling `download-ci-llvm` or using python3")
544 tarball_suffix = '.tar.xz'
545 filename = "rust-dev-nightly-" + self.build + tarball_suffix
546 tarball = os.path.join(rustc_cache, filename)
547 if not os.path.exists(tarball):
548 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
549 unpack(tarball, tarball_suffix, self.llvm_root(),
551 verbose=self.verbose)
553 def fix_bin_or_dylib(self, fname):
554 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
555 or the RPATH section, to fix the dynamic library search path
557 This method is only required on NixOS and uses the PatchELF utility to
558 change the interpreter/RPATH of ELF executables.
560 Please see https://nixos.org/patchelf.html for more information
562 default_encoding = sys.getdefaultencoding()
564 ostype = subprocess.check_output(
565 ['uname', '-s']).strip().decode(default_encoding)
566 except subprocess.CalledProcessError:
568 except OSError as reason:
569 if getattr(reason, 'winerror', None) is not None:
573 if ostype != "Linux":
576 if not os.path.exists("/etc/NIXOS"):
578 if os.path.exists("/lib"):
581 # At this point we're pretty sure the user is running NixOS
582 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
583 print(nix_os_msg, fname)
585 # Only build `.nix-deps` once.
586 nix_deps_dir = self.nix_deps_dir
588 # Run `nix-build` to "build" each dependency (which will likely reuse
589 # the existing `/nix/store` copy, or at most download a pre-built copy).
591 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
592 # directory, but still reference the actual `/nix/store` path in the rpath
593 # as it makes it significantly more robust against changes to the location of
594 # the `.nix-deps` location.
596 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
597 # zlib: Needed as a system dependency of `libLLVM-*.so`.
598 # patchelf: Needed for patching ELF binaries (see doc comment above).
599 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
601 with (import <nixpkgs> {});
603 name = "rust-stage0-dependencies";
612 subprocess.check_output([
613 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
615 except subprocess.CalledProcessError as reason:
616 print("warning: failed to call nix-build:", reason)
618 self.nix_deps_dir = nix_deps_dir
620 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
622 # Relative default, all binary and dynamic libraries we ship
623 # appear to have this (even when `../lib` is redundant).
625 os.path.join(os.path.realpath(nix_deps_dir), "lib")
627 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
628 if not fname.endswith(".so"):
629 # Finally, set the corret .interp for binaries
630 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
631 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
634 subprocess.check_output([patchelf] + patchelf_args + [fname])
635 except subprocess.CalledProcessError as reason:
636 print("warning: failed to call patchelf:", reason)
639 # If `download-rustc` is set, download the most recent commit with CI artifacts
640 def maybe_download_ci_toolchain(self):
641 # If `download-rustc` is not set, default to rebuilding.
642 download_rustc = self.get_toml("download-rustc", section="rust")
643 if download_rustc is None or download_rustc == "false":
645 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
647 # Handle running from a directory other than the top level
648 rev_parse = ["git", "rev-parse", "--show-toplevel"]
649 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
650 compiler = "{}/compiler/".format(top_level)
651 library = "{}/library/".format(top_level)
653 # Look for a version to compare to based on the current commit.
654 # Only commits merged by bors will have CI artifacts.
655 merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"]
656 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
658 # Warn if there were changes to the compiler or standard library since the ancestor commit.
659 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
661 if download_rustc == "if-unchanged":
663 print("warning: `download-rustc` is enabled, but there are changes to \
664 compiler/ or library/")
667 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
668 self.rustc_commit = commit
669 # FIXME: support downloading artifacts from the beta channel
670 self.download_toolchain(False, "nightly")
672 def rustc_stamp(self, stage0):
673 """Return the path for .rustc-stamp at the given stage
676 >>> rb.build_dir = "build"
677 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
679 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
682 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
684 def rustfmt_stamp(self):
685 """Return the path for .rustfmt-stamp
688 >>> rb.build_dir = "build"
689 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
692 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
694 def llvm_stamp(self):
695 """Return the path for .rustfmt-stamp
698 >>> rb.build_dir = "build"
699 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
702 return os.path.join(self.llvm_root(), '.llvm-stamp')
705 def program_out_of_date(self, stamp_path, key):
706 """Check if the given program stamp is out of date"""
707 if not os.path.exists(stamp_path) or self.clean:
709 with open(stamp_path, 'r') as stamp:
710 return key != stamp.read()
712 def bin_root(self, stage0):
713 """Return the binary root directory for the given stage
716 >>> rb.build_dir = "build"
717 >>> rb.bin_root(True) == os.path.join("build", "stage0")
719 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
722 When the 'build' property is given should be a nested directory:
724 >>> rb.build = "devel"
725 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
732 return os.path.join(self.build_dir, self.build, subdir)
735 """Return the CI LLVM root directory
738 >>> rb.build_dir = "build"
739 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
742 When the 'build' property is given should be a nested directory:
744 >>> rb.build = "devel"
745 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
748 return os.path.join(self.build_dir, self.build, "ci-llvm")
750 def get_toml(self, key, section=None):
751 """Returns the value of the given key in config.toml, otherwise returns None
754 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
755 >>> rb.get_toml("key2")
758 If the key does not exists, the result is None:
760 >>> rb.get_toml("key3") is None
763 Optionally also matches the section the key appears in
765 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
766 >>> rb.get_toml('key', 'a')
768 >>> rb.get_toml('key', 'b')
770 >>> rb.get_toml('key', 'c') is None
773 >>> rb.config_toml = 'key1 = true'
774 >>> rb.get_toml("key1")
779 for line in self.config_toml.splitlines():
780 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
781 if section_match is not None:
782 cur_section = section_match.group(1)
784 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
785 if match is not None:
786 value = match.group(1)
787 if section is None or section == cur_section:
788 return self.get_string(value) or value.strip()
792 """Return config path for cargo"""
793 return self.program_config('cargo')
795 def rustc(self, stage0):
796 """Return config path for rustc"""
797 return self.program_config('rustc', stage0)
800 """Return config path for rustfmt"""
801 if not self.rustfmt_channel:
803 return self.program_config('rustfmt')
805 def program_config(self, program, stage0=True):
806 """Return config path for the given program at the given stage
809 >>> rb.config_toml = 'rustc = "rustc"\\n'
810 >>> rb.program_config('rustc')
812 >>> rb.config_toml = ''
813 >>> cargo_path = rb.program_config('cargo', True)
814 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
817 >>> cargo_path = rb.program_config('cargo', False)
818 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
822 config = self.get_toml(program)
824 return os.path.expanduser(config)
825 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
826 program, self.exe_suffix()))
829 def get_string(line):
830 """Return the value between double quotes
832 >>> RustBuild.get_string(' "devel" ')
834 >>> RustBuild.get_string(" 'devel' ")
836 >>> RustBuild.get_string('devel') is None
838 >>> RustBuild.get_string(' "devel ')
841 start = line.find('"')
843 end = start + 1 + line[start + 1:].find('"')
844 return line[start + 1:end]
845 start = line.find('\'')
847 end = start + 1 + line[start + 1:].find('\'')
848 return line[start + 1:end]
853 """Return a suffix for executables"""
854 if sys.platform == 'win32':
858 def bootstrap_binary(self):
859 """Return the path of the bootstrap binary
862 >>> rb.build_dir = "build"
863 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
864 ... "debug", "bootstrap")
867 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
869 def build_bootstrap(self):
870 """Build bootstrap"""
871 build_dir = os.path.join(self.build_dir, "bootstrap")
872 if self.clean and os.path.exists(build_dir):
873 shutil.rmtree(build_dir)
874 env = os.environ.copy()
875 # `CARGO_BUILD_TARGET` breaks bootstrap build.
876 # See also: <https://github.com/rust-lang/rust/issues/70208>.
877 if "CARGO_BUILD_TARGET" in env:
878 del env["CARGO_BUILD_TARGET"]
879 env["CARGO_TARGET_DIR"] = build_dir
880 env["RUSTC"] = self.rustc(True)
881 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
882 (os.pathsep + env["LD_LIBRARY_PATH"]) \
883 if "LD_LIBRARY_PATH" in env else ""
884 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
885 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
886 if "DYLD_LIBRARY_PATH" in env else ""
887 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
888 (os.pathsep + env["LIBRARY_PATH"]) \
889 if "LIBRARY_PATH" in env else ""
890 # preserve existing RUSTFLAGS
891 env.setdefault("RUSTFLAGS", "")
892 env["RUSTFLAGS"] += " -Cdebuginfo=2"
894 build_section = "target.{}".format(self.build)
896 if self.get_toml("crt-static", build_section) == "true":
897 target_features += ["+crt-static"]
898 elif self.get_toml("crt-static", build_section) == "false":
899 target_features += ["-crt-static"]
901 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
902 target_linker = self.get_toml("linker", build_section)
903 if target_linker is not None:
904 env["RUSTFLAGS"] += " -C linker=" + target_linker
905 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
906 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
907 if self.get_toml("deny-warnings", "rust") != "false":
908 env["RUSTFLAGS"] += " -Dwarnings"
910 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
911 os.pathsep + env["PATH"]
912 if not os.path.isfile(self.cargo()):
913 raise Exception("no cargo executable found at `{}`".format(
915 args = [self.cargo(), "build", "--manifest-path",
916 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
917 for _ in range(1, self.verbose):
918 args.append("--verbose")
919 if self.use_locked_deps:
920 args.append("--locked")
921 if self.use_vendored_sources:
922 args.append("--frozen")
923 run(args, env=env, verbose=self.verbose)
925 def build_triple(self):
926 """Build triple as in LLVM
928 Note that `default_build_triple` is moderately expensive,
929 so use `self.build` where possible.
931 config = self.get_toml('build')
934 return default_build_triple(self.verbose)
936 def check_submodule(self, module, slow_submodules):
937 if not slow_submodules:
938 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
939 cwd=os.path.join(self.rust_root, module),
940 stdout=subprocess.PIPE)
945 def update_submodule(self, module, checked_out, recorded_submodules):
946 module_path = os.path.join(self.rust_root, module)
948 if checked_out is not None:
949 default_encoding = sys.getdefaultencoding()
950 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
951 if recorded_submodules[module] == checked_out:
954 print("Updating submodule", module)
956 run(["git", "submodule", "-q", "sync", module],
957 cwd=self.rust_root, verbose=self.verbose)
959 update_args = ["git", "submodule", "update", "--init", "--recursive"]
960 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
961 update_args.append("--progress")
962 update_args.append(module)
963 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
965 run(["git", "reset", "-q", "--hard"],
966 cwd=module_path, verbose=self.verbose)
967 run(["git", "clean", "-qdfx"],
968 cwd=module_path, verbose=self.verbose)
970 def update_submodules(self):
971 """Update submodules"""
972 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
973 self.get_toml('submodules') == "false":
976 default_encoding = sys.getdefaultencoding()
978 # check the existence and version of 'git' command
979 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
980 self.git_version = distutils.version.LooseVersion(git_version_str)
982 slow_submodules = self.get_toml('fast-submodules') == "false"
985 print('Unconditionally updating all submodules')
987 print('Updating only changed submodules')
988 default_encoding = sys.getdefaultencoding()
989 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
990 ["git", "config", "--file",
991 os.path.join(self.rust_root, ".gitmodules"),
992 "--get-regexp", "path"]
993 ).decode(default_encoding).splitlines()]
994 filtered_submodules = []
995 submodules_names = []
996 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
997 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
998 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
999 or "llvm" in self.get_toml('codegen-backends', 'rust')
1000 for module in submodules:
1001 if module.endswith("llvm-project"):
1002 # Don't sync the llvm-project submodule if an external LLVM was
1003 # provided, if we are downloading LLVM or if the LLVM backend is
1004 # not being built. Also, if the submodule has been initialized
1005 # already, sync it anyways so that it doesn't mess up contributor
1007 if external_llvm_provided or not llvm_needed:
1008 if self.get_toml('lld') != 'true' and not llvm_checked_out:
1010 check = self.check_submodule(module, slow_submodules)
1011 filtered_submodules.append((module, check))
1012 submodules_names.append(module)
1013 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1014 cwd=self.rust_root, stdout=subprocess.PIPE)
1015 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1016 recorded_submodules = {}
1017 for data in recorded:
1019 recorded_submodules[data[3]] = data[2]
1020 for module in filtered_submodules:
1021 self.update_submodule(module[0], module[1], recorded_submodules)
1022 print("Submodules updated in %.2f seconds" % (time() - start_time))
1024 def set_normal_environment(self):
1025 """Set download URL for normal environment"""
1026 if 'RUSTUP_DIST_SERVER' in os.environ:
1027 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1029 self._download_url = 'https://static.rust-lang.org'
1031 def set_dev_environment(self):
1032 """Set download URL for development environment"""
1033 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1034 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1036 self._download_url = 'https://dev-static.rust-lang.org'
1038 def check_vendored_status(self):
1039 """Check that vendoring is configured properly"""
1040 vendor_dir = os.path.join(self.rust_root, 'vendor')
1041 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1042 if os.environ.get('USER') != os.environ['SUDO_USER']:
1043 self.use_vendored_sources = True
1044 print('info: looks like you are running this command under `sudo`')
1045 print(' and so in order to preserve your $HOME this will now')
1046 print(' use vendored sources by default.')
1047 if not os.path.exists(vendor_dir):
1048 print('error: vendoring required, but vendor directory does not exist.')
1049 print(' Run `cargo vendor` without sudo to initialize the '
1050 'vendor directory.')
1051 raise Exception("{} not found".format(vendor_dir))
1053 if self.use_vendored_sources:
1054 if not os.path.exists('.cargo'):
1055 os.makedirs('.cargo')
1056 with output('.cargo/config') as cargo_config:
1058 "[source.crates-io]\n"
1059 "replace-with = 'vendored-sources'\n"
1060 "registry = 'https://example.com'\n"
1062 "[source.vendored-sources]\n"
1063 "directory = '{}/vendor'\n"
1064 .format(self.rust_root))
1066 if os.path.exists('.cargo'):
1067 shutil.rmtree('.cargo')
1069 def ensure_vendored(self):
1070 """Ensure that the vendored sources are available if needed"""
1071 vendor_dir = os.path.join(self.rust_root, 'vendor')
1072 # Note that this does not handle updating the vendored dependencies if
1073 # the rust git repository is updated. Normal development usually does
1074 # not use vendoring, so hopefully this isn't too much of a problem.
1075 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1079 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1080 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1081 ], verbose=self.verbose, cwd=self.rust_root)
1084 def bootstrap(help_triggered):
1085 """Configure, fetch, build and run the initial bootstrap"""
1087 # If the user is asking for help, let them know that the whole download-and-build
1088 # process has to happen before anything is printed out.
1090 print("info: Downloading and building bootstrap before processing --help")
1091 print(" command. See src/bootstrap/README.md for help with common")
1094 parser = argparse.ArgumentParser(description='Build rust')
1095 parser.add_argument('--config')
1096 parser.add_argument('--build')
1097 parser.add_argument('--clean', action='store_true')
1098 parser.add_argument('-v', '--verbose', action='count', default=0)
1100 args = [a for a in sys.argv if a != '-h' and a != '--help']
1101 args, _ = parser.parse_known_args(args)
1103 # Configure initial bootstrap
1105 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1106 build.verbose = args.verbose
1107 build.clean = args.clean
1109 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1111 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1112 if not toml_path and os.path.exists('config.toml'):
1113 toml_path = 'config.toml'
1116 if not os.path.exists(toml_path):
1117 toml_path = os.path.join(build.rust_root, toml_path)
1119 with open(toml_path) as config:
1120 build.config_toml = config.read()
1122 profile = build.get_toml('profile')
1123 if profile is not None:
1124 include_file = 'config.{}.toml'.format(profile)
1125 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1126 include_path = os.path.join(include_dir, include_file)
1127 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1128 # specific key, so appending our defaults at the end allows the user to override them
1129 with open(include_path) as included_toml:
1130 build.config_toml += os.linesep + included_toml.read()
1132 config_verbose = build.get_toml('verbose', 'build')
1133 if config_verbose is not None:
1134 build.verbose = max(build.verbose, int(config_verbose))
1136 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1138 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1140 build.check_vendored_status()
1142 build_dir = build.get_toml('build-dir', 'build') or 'build'
1143 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1145 data = stage0_data(build.rust_root)
1146 build.date = data['date']
1147 build.rustc_channel = data['rustc']
1149 if "rustfmt" in data:
1150 build.rustfmt_channel = data['rustfmt']
1153 build.set_dev_environment()
1155 build.set_normal_environment()
1157 build.build = args.build or build.build_triple()
1158 build.update_submodules()
1160 # Fetch/build the bootstrap
1161 build.download_toolchain()
1162 # Download the master compiler if `download-rustc` is set
1163 build.maybe_download_ci_toolchain()
1165 build.ensure_vendored()
1166 build.build_bootstrap()
1170 args = [build.bootstrap_binary()]
1171 args.extend(sys.argv[1:])
1172 env = os.environ.copy()
1173 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1174 env["BOOTSTRAP_PYTHON"] = sys.executable
1175 env["BUILD_DIR"] = build.build_dir
1176 env["RUSTC_BOOTSTRAP"] = '1'
1178 env["BOOTSTRAP_CONFIG"] = toml_path
1179 if build.rustc_commit is not None:
1180 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1181 run(args, env=env, verbose=build.verbose)
1185 """Entry point for the bootstrap process"""
1188 # x.py help <cmd> ...
1189 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1190 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1193 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1195 bootstrap(help_triggered)
1196 if not help_triggered:
1197 print("Build completed successfully in {}".format(
1198 format_build_time(time() - start_time)))
1199 except (SystemExit, KeyboardInterrupt) as error:
1200 if hasattr(error, 'code') and isinstance(error.code, int):
1201 exit_code = error.code
1205 if not help_triggered:
1206 print("Build completed unsuccessfully in {}".format(
1207 format_build_time(time() - start_time)))
1211 if __name__ == '__main__':