1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"],
198 stderr=subprocess.DEVNULL)
199 version = version.decode(default_encoding)
200 host = next(x for x in version.split('\n') if x.startswith("host: "))
201 triple = host.split("host: ")[1]
203 print("detected default triple {}".format(triple))
205 except Exception as e:
207 print("rustup not detected: {}".format(e))
208 print("falling back to auto-detect")
210 required = sys.platform != 'win32'
211 ostype = require(["uname", "-s"], exit=required)
212 cputype = require(['uname', '-m'], exit=required)
214 # If we do not have `uname`, assume Windows.
215 if ostype is None or cputype is None:
216 return 'x86_64-pc-windows-msvc'
218 ostype = ostype.decode(default_encoding)
219 cputype = cputype.decode(default_encoding)
221 # The goal here is to come up with the same triple as LLVM would,
222 # at least for the subset of platforms we're willing to target.
224 'Darwin': 'apple-darwin',
225 'DragonFly': 'unknown-dragonfly',
226 'FreeBSD': 'unknown-freebsd',
227 'Haiku': 'unknown-haiku',
228 'NetBSD': 'unknown-netbsd',
229 'OpenBSD': 'unknown-openbsd'
232 # Consider the direct transformation first and then the special cases
233 if ostype in ostype_mapper:
234 ostype = ostype_mapper[ostype]
235 elif ostype == 'Linux':
236 os_from_sp = subprocess.check_output(
237 ['uname', '-o']).strip().decode(default_encoding)
238 if os_from_sp == 'Android':
239 ostype = 'linux-android'
241 ostype = 'unknown-linux-gnu'
242 elif ostype == 'SunOS':
243 ostype = 'pc-solaris'
244 # On Solaris, uname -m will return a machine classification instead
245 # of a cpu type, so uname -p is recommended instead. However, the
246 # output from that option is too generic for our purposes (it will
247 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
248 # must be used instead.
249 cputype = require(['isainfo', '-k']).decode(default_encoding)
250 # sparc cpus have sun as a target vendor
251 if 'sparc' in cputype:
252 ostype = 'sun-solaris'
253 elif ostype.startswith('MINGW'):
254 # msys' `uname` does not print gcc configuration, but prints msys
255 # configuration. so we cannot believe `uname -m`:
256 # msys1 is always i686 and msys2 is always x86_64.
257 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
259 ostype = 'pc-windows-gnu'
261 if os.environ.get('MSYSTEM') == 'MINGW64':
263 elif ostype.startswith('MSYS'):
264 ostype = 'pc-windows-gnu'
265 elif ostype.startswith('CYGWIN_NT'):
267 if ostype.endswith('WOW64'):
269 ostype = 'pc-windows-gnu'
270 elif sys.platform == 'win32':
271 # Some Windows platforms might have a `uname` command that returns a
272 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
273 # these cases, fall back to using sys.platform.
274 return 'x86_64-pc-windows-msvc'
276 err = "unknown OS type: {}".format(ostype)
279 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
280 cputype = subprocess.check_output(
281 ['uname', '-p']).strip().decode(default_encoding)
284 'aarch64': 'aarch64',
291 'powerpc': 'powerpc',
292 'powerpc64': 'powerpc64',
293 'powerpc64le': 'powerpc64le',
295 'ppc64': 'powerpc64',
296 'ppc64le': 'powerpc64le',
304 # Consider the direct transformation first and then the special cases
305 if cputype in cputype_mapper:
306 cputype = cputype_mapper[cputype]
307 elif cputype in {'xscale', 'arm'}:
309 if ostype == 'linux-android':
310 ostype = 'linux-androideabi'
311 elif ostype == 'unknown-freebsd':
312 cputype = subprocess.check_output(
313 ['uname', '-p']).strip().decode(default_encoding)
314 ostype = 'unknown-freebsd'
315 elif cputype == 'armv6l':
317 if ostype == 'linux-android':
318 ostype = 'linux-androideabi'
321 elif cputype in {'armv7l', 'armv8l'}:
323 if ostype == 'linux-android':
324 ostype = 'linux-androideabi'
327 elif cputype == 'mips':
328 if sys.byteorder == 'big':
330 elif sys.byteorder == 'little':
333 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
334 elif cputype == 'mips64':
335 if sys.byteorder == 'big':
337 elif sys.byteorder == 'little':
340 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
341 # only the n64 ABI is supported, indicate it
343 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
346 err = "unknown cpu type: {}".format(cputype)
349 return "{}-{}".format(cputype, ostype)
352 @contextlib.contextmanager
353 def output(filepath):
354 tmp = filepath + '.tmp'
355 with open(tmp, 'w') as f:
358 if os.path.exists(filepath):
359 os.remove(filepath) # PermissionError/OSError on Win32 if in use
361 shutil.copy2(tmp, filepath)
364 os.rename(tmp, filepath)
367 class RustBuild(object):
368 """Provide all the methods required to build Rust"""
371 self._download_url = ''
372 self.rustc_channel = ''
373 self.rustfmt_channel = ''
377 self.config_toml = ''
379 self.use_locked_deps = ''
380 self.use_vendored_sources = ''
382 self.git_version = None
383 self.nix_deps_dir = None
384 self.rustc_commit = None
386 def download_stage0(self):
387 """Fetch the build system for Rust, written in Rust
389 This method will build a cache directory, then it will fetch the
390 tarball which has the stage0 compiler used to then bootstrap the Rust
393 Each downloaded tarball is extracted, after that, the script
394 will move all the content to the right place.
396 rustc_channel = self.rustc_channel
397 rustfmt_channel = self.rustfmt_channel
399 if self.rustc().startswith(self.bin_root()) and \
400 (not os.path.exists(self.rustc()) or
401 self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
402 if os.path.exists(self.bin_root()):
403 shutil.rmtree(self.bin_root())
404 download_rustc = self.rustc_commit is not None
405 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
406 filename = "rust-std-{}-{}{}".format(
407 rustc_channel, self.build, tarball_suffix)
408 pattern = "rust-std-{}".format(self.build)
409 self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
410 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
412 self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
413 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
415 self._download_component_helper(filename, "cargo", tarball_suffix)
416 if self.rustc_commit is not None:
417 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
418 self._download_component_helper(
419 filename, "rustc-dev", tarball_suffix, download_rustc
422 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
423 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
424 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
425 lib_dir = "{}/lib".format(self.bin_root())
426 for lib in os.listdir(lib_dir):
427 if lib.endswith(".so"):
428 self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
429 with output(self.rustc_stamp()) as rust_stamp:
430 rust_stamp.write(self.date + str(self.rustc_commit))
432 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
433 not os.path.exists(self.rustfmt())
434 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
437 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
438 [channel, date] = rustfmt_channel.split('-', 1)
439 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
440 self._download_component_helper(
441 filename, "rustfmt-preview", tarball_suffix, key=date
443 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
444 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
445 with output(self.rustfmt_stamp()) as rustfmt_stamp:
446 rustfmt_stamp.write(self.rustfmt_channel)
448 if self.downloading_llvm():
449 # We want the most recent LLVM submodule update to avoid downloading
450 # LLVM more often than necessary.
452 # This git command finds that commit SHA, looking for bors-authored
453 # merges that modified src/llvm-project.
455 # This works even in a repository that has not yet initialized
457 top_level = subprocess.check_output([
458 "git", "rev-parse", "--show-toplevel",
459 ]).decode(sys.getdefaultencoding()).strip()
460 llvm_sha = subprocess.check_output([
461 "git", "log", "--author=bors", "--format=%H", "-n1",
462 "-m", "--first-parent",
464 "{}/src/llvm-project".format(top_level),
465 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
466 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
467 "{}/src/version".format(top_level)
468 ]).decode(sys.getdefaultencoding()).strip()
469 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
470 llvm_root = self.llvm_root()
471 llvm_lib = os.path.join(llvm_root, "lib")
472 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
473 self._download_ci_llvm(llvm_sha, llvm_assertions)
474 for binary in ["llvm-config", "FileCheck"]:
475 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
476 for lib in os.listdir(llvm_lib):
477 if lib.endswith(".so"):
478 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
479 with output(self.llvm_stamp()) as llvm_stamp:
480 llvm_stamp.write(llvm_sha + str(llvm_assertions))
482 def downloading_llvm(self):
483 opt = self.get_toml('download-ci-llvm', 'llvm')
484 # This is currently all tier 1 targets (since others may not have CI
486 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
487 supported_platforms = [
488 "aarch64-unknown-linux-gnu",
489 "i686-pc-windows-gnu",
490 "i686-pc-windows-msvc",
491 "i686-unknown-linux-gnu",
492 "x86_64-unknown-linux-gnu",
493 "x86_64-apple-darwin",
494 "x86_64-pc-windows-gnu",
495 "x86_64-pc-windows-msvc",
497 return opt == "true" \
498 or (opt == "if-available" and self.build in supported_platforms)
500 def _download_component_helper(
501 self, filename, pattern, tarball_suffix, download_rustc=False, key=None
505 key = self.rustc_commit
508 cache_dst = os.path.join(self.build_dir, "cache")
509 rustc_cache = os.path.join(cache_dst, key)
510 if not os.path.exists(rustc_cache):
511 os.makedirs(rustc_cache)
514 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
516 url = "{}/dist/{}".format(self._download_url, key)
517 tarball = os.path.join(rustc_cache, filename)
518 if not os.path.exists(tarball):
519 do_verify = not download_rustc
520 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
521 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
523 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
524 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
525 cache_dst = os.path.join(self.build_dir, "cache")
526 rustc_cache = os.path.join(cache_dst, cache_prefix)
527 if not os.path.exists(rustc_cache):
528 os.makedirs(rustc_cache)
530 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
532 url = url.replace('rustc-builds', 'rustc-builds-alt')
533 # ci-artifacts are only stored as .xz, not .gz
535 print("error: XZ support is required to download LLVM")
536 print("help: consider disabling `download-ci-llvm` or using python3")
538 tarball_suffix = '.tar.xz'
539 filename = "rust-dev-nightly-" + self.build + tarball_suffix
540 tarball = os.path.join(rustc_cache, filename)
541 if not os.path.exists(tarball):
542 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
543 unpack(tarball, tarball_suffix, self.llvm_root(),
545 verbose=self.verbose)
547 def fix_bin_or_dylib(self, fname, rpath_libz=False):
548 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
549 or the RPATH section, to fix the dynamic library search path
551 This method is only required on NixOS and uses the PatchELF utility to
552 change the interpreter/RPATH of ELF executables.
554 Please see https://nixos.org/patchelf.html for more information
556 default_encoding = sys.getdefaultencoding()
558 ostype = subprocess.check_output(
559 ['uname', '-s']).strip().decode(default_encoding)
560 except subprocess.CalledProcessError:
562 except OSError as reason:
563 if getattr(reason, 'winerror', None) is not None:
567 if ostype != "Linux":
570 if not os.path.exists("/etc/NIXOS"):
572 if os.path.exists("/lib"):
575 # At this point we're pretty sure the user is running NixOS
576 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
577 print(nix_os_msg, fname)
579 # Only build `stage0/.nix-deps` once.
580 nix_deps_dir = self.nix_deps_dir
582 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
583 if not os.path.exists(nix_deps_dir):
584 os.makedirs(nix_deps_dir)
587 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
588 "stdenv.cc.bintools",
590 # Needed as a system dependency of `libLLVM-*.so`.
593 # Needed for patching ELF binaries (see doc comment above).
597 # Run `nix-build` to "build" each dependency (which will likely reuse
598 # the existing `/nix/store` copy, or at most download a pre-built copy).
599 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
600 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
601 # ensuring garbage collection will never remove the `/nix/store` path
602 # (which would break our patched binaries that hardcode those paths).
605 subprocess.check_output([
606 "nix-build", "<nixpkgs>",
608 "-o", "{}/{}".format(nix_deps_dir, dep),
610 except subprocess.CalledProcessError as reason:
611 print("warning: failed to call nix-build:", reason)
614 self.nix_deps_dir = nix_deps_dir
616 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
620 # Patch RPATH to add `zlib` dependency that stems from LLVM
621 dylib_deps = ["zlib"]
623 # Relative default, all binary and dynamic libraries we ship
624 # appear to have this (even when `../lib` is redundant).
626 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
627 patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
628 if not fname.endswith(".so"):
629 # Finally, set the corret .interp for binaries
630 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
631 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
632 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
635 subprocess.check_output([patchelf] + patchelf_args + [fname])
636 except subprocess.CalledProcessError as reason:
637 print("warning: failed to call patchelf:", reason)
640 # Return the stage1 compiler to download, if any.
641 def maybe_download_rustc(self):
642 # If `download-rustc` is not set, default to rebuilding.
643 if self.get_toml("download-rustc", section="rust") != "true":
646 # Handle running from a directory other than the top level
647 rev_parse = ["git", "rev-parse", "--show-toplevel"]
648 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
649 compiler = "{}/compiler/".format(top_level)
651 # Look for a version to compare to based on the current commit.
652 # Only commits merged by bors will have CI artifacts.
653 merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"]
654 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
656 # Warn if there were changes to the compiler since the ancestor commit.
657 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
659 print("warning: `download-rustc` is enabled, but there are changes to compiler/")
663 def rustc_stamp(self):
664 """Return the path for .rustc-stamp
667 >>> rb.build_dir = "build"
668 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
671 return os.path.join(self.bin_root(), '.rustc-stamp')
673 def rustfmt_stamp(self):
674 """Return the path for .rustfmt-stamp
677 >>> rb.build_dir = "build"
678 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
681 return os.path.join(self.bin_root(), '.rustfmt-stamp')
683 def llvm_stamp(self):
684 """Return the path for .rustfmt-stamp
687 >>> rb.build_dir = "build"
688 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
691 return os.path.join(self.llvm_root(), '.llvm-stamp')
694 def program_out_of_date(self, stamp_path, key):
695 """Check if the given program stamp is out of date"""
696 if not os.path.exists(stamp_path) or self.clean:
698 with open(stamp_path, 'r') as stamp:
699 return key != stamp.read()
702 """Return the binary root directory
705 >>> rb.build_dir = "build"
706 >>> rb.bin_root() == os.path.join("build", "stage0")
709 When the 'build' property is given should be a nested directory:
711 >>> rb.build = "devel"
712 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
715 return os.path.join(self.build_dir, self.build, "stage0")
718 """Return the CI LLVM root directory
721 >>> rb.build_dir = "build"
722 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
725 When the 'build' property is given should be a nested directory:
727 >>> rb.build = "devel"
728 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
731 return os.path.join(self.build_dir, self.build, "ci-llvm")
733 def get_toml(self, key, section=None):
734 """Returns the value of the given key in config.toml, otherwise returns None
737 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
738 >>> rb.get_toml("key2")
741 If the key does not exists, the result is None:
743 >>> rb.get_toml("key3") is None
746 Optionally also matches the section the key appears in
748 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
749 >>> rb.get_toml('key', 'a')
751 >>> rb.get_toml('key', 'b')
753 >>> rb.get_toml('key', 'c') is None
756 >>> rb.config_toml = 'key1 = true'
757 >>> rb.get_toml("key1")
762 for line in self.config_toml.splitlines():
763 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
764 if section_match is not None:
765 cur_section = section_match.group(1)
767 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
768 if match is not None:
769 value = match.group(1)
770 if section is None or section == cur_section:
771 return self.get_string(value) or value.strip()
775 """Return config path for cargo"""
776 return self.program_config('cargo')
779 """Return config path for rustc"""
780 return self.program_config('rustc')
783 """Return config path for rustfmt"""
784 if not self.rustfmt_channel:
786 return self.program_config('rustfmt')
788 def program_config(self, program):
789 """Return config path for the given program
792 >>> rb.config_toml = 'rustc = "rustc"\\n'
793 >>> rb.program_config('rustc')
795 >>> rb.config_toml = ''
796 >>> cargo_path = rb.program_config('cargo')
797 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
801 config = self.get_toml(program)
803 return os.path.expanduser(config)
804 return os.path.join(self.bin_root(), "bin", "{}{}".format(
805 program, self.exe_suffix()))
808 def get_string(line):
809 """Return the value between double quotes
811 >>> RustBuild.get_string(' "devel" ')
813 >>> RustBuild.get_string(" 'devel' ")
815 >>> RustBuild.get_string('devel') is None
817 >>> RustBuild.get_string(' "devel ')
820 start = line.find('"')
822 end = start + 1 + line[start + 1:].find('"')
823 return line[start + 1:end]
824 start = line.find('\'')
826 end = start + 1 + line[start + 1:].find('\'')
827 return line[start + 1:end]
832 """Return a suffix for executables"""
833 if sys.platform == 'win32':
837 def bootstrap_binary(self):
838 """Return the path of the bootstrap binary
841 >>> rb.build_dir = "build"
842 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
843 ... "debug", "bootstrap")
846 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
848 def build_bootstrap(self):
849 """Build bootstrap"""
850 build_dir = os.path.join(self.build_dir, "bootstrap")
851 if self.clean and os.path.exists(build_dir):
852 shutil.rmtree(build_dir)
853 env = os.environ.copy()
854 # `CARGO_BUILD_TARGET` breaks bootstrap build.
855 # See also: <https://github.com/rust-lang/rust/issues/70208>.
856 if "CARGO_BUILD_TARGET" in env:
857 del env["CARGO_BUILD_TARGET"]
858 env["CARGO_TARGET_DIR"] = build_dir
859 env["RUSTC"] = self.rustc()
860 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
861 (os.pathsep + env["LD_LIBRARY_PATH"]) \
862 if "LD_LIBRARY_PATH" in env else ""
863 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
864 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
865 if "DYLD_LIBRARY_PATH" in env else ""
866 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
867 (os.pathsep + env["LIBRARY_PATH"]) \
868 if "LIBRARY_PATH" in env else ""
869 # preserve existing RUSTFLAGS
870 env.setdefault("RUSTFLAGS", "")
871 env["RUSTFLAGS"] += " -Cdebuginfo=2"
873 build_section = "target.{}".format(self.build)
875 if self.get_toml("crt-static", build_section) == "true":
876 target_features += ["+crt-static"]
877 elif self.get_toml("crt-static", build_section) == "false":
878 target_features += ["-crt-static"]
880 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
881 target_linker = self.get_toml("linker", build_section)
882 if target_linker is not None:
883 env["RUSTFLAGS"] += " -C linker=" + target_linker
884 # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump
885 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
886 if self.get_toml("deny-warnings", "rust") != "false":
887 env["RUSTFLAGS"] += " -Dwarnings"
889 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
890 os.pathsep + env["PATH"]
891 if not os.path.isfile(self.cargo()):
892 raise Exception("no cargo executable found at `{}`".format(
894 args = [self.cargo(), "build", "--manifest-path",
895 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
896 for _ in range(1, self.verbose):
897 args.append("--verbose")
898 if self.use_locked_deps:
899 args.append("--locked")
900 if self.use_vendored_sources:
901 args.append("--frozen")
902 run(args, env=env, verbose=self.verbose)
904 def build_triple(self):
905 """Build triple as in LLVM
907 Note that `default_build_triple` is moderately expensive,
908 so use `self.build` where possible.
910 config = self.get_toml('build')
913 return default_build_triple(self.verbose)
915 def check_submodule(self, module, slow_submodules):
916 if not slow_submodules:
917 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
918 cwd=os.path.join(self.rust_root, module),
919 stdout=subprocess.PIPE)
924 def update_submodule(self, module, checked_out, recorded_submodules):
925 module_path = os.path.join(self.rust_root, module)
927 if checked_out is not None:
928 default_encoding = sys.getdefaultencoding()
929 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
930 if recorded_submodules[module] == checked_out:
933 print("Updating submodule", module)
935 run(["git", "submodule", "-q", "sync", module],
936 cwd=self.rust_root, verbose=self.verbose)
938 update_args = ["git", "submodule", "update", "--init", "--recursive"]
939 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
940 update_args.append("--progress")
941 update_args.append(module)
942 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
944 run(["git", "reset", "-q", "--hard"],
945 cwd=module_path, verbose=self.verbose)
946 run(["git", "clean", "-qdfx"],
947 cwd=module_path, verbose=self.verbose)
949 def update_submodules(self):
950 """Update submodules"""
951 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
952 self.get_toml('submodules') == "false":
955 default_encoding = sys.getdefaultencoding()
957 # check the existence and version of 'git' command
958 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
959 self.git_version = distutils.version.LooseVersion(git_version_str)
961 slow_submodules = self.get_toml('fast-submodules') == "false"
964 print('Unconditionally updating all submodules')
966 print('Updating only changed submodules')
967 default_encoding = sys.getdefaultencoding()
968 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
969 ["git", "config", "--file",
970 os.path.join(self.rust_root, ".gitmodules"),
971 "--get-regexp", "path"]
972 ).decode(default_encoding).splitlines()]
973 filtered_submodules = []
974 submodules_names = []
975 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
976 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
977 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
978 or "llvm" in self.get_toml('codegen-backends', 'rust')
979 for module in submodules:
980 if module.endswith("llvm-project"):
981 # Don't sync the llvm-project submodule if an external LLVM was
982 # provided, if we are downloading LLVM or if the LLVM backend is
983 # not being built. Also, if the submodule has been initialized
984 # already, sync it anyways so that it doesn't mess up contributor
986 if external_llvm_provided or not llvm_needed:
987 if self.get_toml('lld') != 'true' and not llvm_checked_out:
989 check = self.check_submodule(module, slow_submodules)
990 filtered_submodules.append((module, check))
991 submodules_names.append(module)
992 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
993 cwd=self.rust_root, stdout=subprocess.PIPE)
994 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
995 recorded_submodules = {}
996 for data in recorded:
998 recorded_submodules[data[3]] = data[2]
999 for module in filtered_submodules:
1000 self.update_submodule(module[0], module[1], recorded_submodules)
1001 print("Submodules updated in %.2f seconds" % (time() - start_time))
1003 def set_normal_environment(self):
1004 """Set download URL for normal environment"""
1005 if 'RUSTUP_DIST_SERVER' in os.environ:
1006 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1008 self._download_url = 'https://static.rust-lang.org'
1010 def set_dev_environment(self):
1011 """Set download URL for development environment"""
1012 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1013 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1015 self._download_url = 'https://dev-static.rust-lang.org'
1017 def check_vendored_status(self):
1018 """Check that vendoring is configured properly"""
1019 vendor_dir = os.path.join(self.rust_root, 'vendor')
1020 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1021 if os.environ.get('USER') != os.environ['SUDO_USER']:
1022 self.use_vendored_sources = True
1023 print('info: looks like you are running this command under `sudo`')
1024 print(' and so in order to preserve your $HOME this will now')
1025 print(' use vendored sources by default.')
1026 if not os.path.exists(vendor_dir):
1027 print('error: vendoring required, but vendor directory does not exist.')
1028 print(' Run `cargo vendor` without sudo to initialize the '
1029 'vendor directory.')
1030 raise Exception("{} not found".format(vendor_dir))
1032 if self.use_vendored_sources:
1033 if not os.path.exists('.cargo'):
1034 os.makedirs('.cargo')
1035 with output('.cargo/config') as cargo_config:
1037 "[source.crates-io]\n"
1038 "replace-with = 'vendored-sources'\n"
1039 "registry = 'https://example.com'\n"
1041 "[source.vendored-sources]\n"
1042 "directory = '{}/vendor'\n"
1043 .format(self.rust_root))
1045 if os.path.exists('.cargo'):
1046 shutil.rmtree('.cargo')
1048 def ensure_vendored(self):
1049 """Ensure that the vendored sources are available if needed"""
1050 vendor_dir = os.path.join(self.rust_root, 'vendor')
1051 # Note that this does not handle updating the vendored dependencies if
1052 # the rust git repository is updated. Normal development usually does
1053 # not use vendoring, so hopefully this isn't too much of a problem.
1054 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1058 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1059 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1060 ], verbose=self.verbose, cwd=self.rust_root)
1063 def bootstrap(help_triggered):
1064 """Configure, fetch, build and run the initial bootstrap"""
1066 # If the user is asking for help, let them know that the whole download-and-build
1067 # process has to happen before anything is printed out.
1069 print("info: Downloading and building bootstrap before processing --help")
1070 print(" command. See src/bootstrap/README.md for help with common")
1073 parser = argparse.ArgumentParser(description='Build rust')
1074 parser.add_argument('--config')
1075 parser.add_argument('--build')
1076 parser.add_argument('--clean', action='store_true')
1077 parser.add_argument('-v', '--verbose', action='count', default=0)
1079 args = [a for a in sys.argv if a != '-h' and a != '--help']
1080 args, _ = parser.parse_known_args(args)
1082 # Configure initial bootstrap
1084 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1085 build.verbose = args.verbose
1086 build.clean = args.clean
1088 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1090 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1091 if not toml_path and os.path.exists('config.toml'):
1092 toml_path = 'config.toml'
1095 if not os.path.exists(toml_path):
1096 toml_path = os.path.join(build.rust_root, toml_path)
1098 with open(toml_path) as config:
1099 build.config_toml = config.read()
1101 profile = build.get_toml('profile')
1102 if profile is not None:
1103 include_file = 'config.{}.toml'.format(profile)
1104 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1105 include_path = os.path.join(include_dir, include_file)
1106 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1107 # specific key, so appending our defaults at the end allows the user to override them
1108 with open(include_path) as included_toml:
1109 build.config_toml += os.linesep + included_toml.read()
1111 config_verbose = build.get_toml('verbose', 'build')
1112 if config_verbose is not None:
1113 build.verbose = max(build.verbose, int(config_verbose))
1115 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1117 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1119 build.check_vendored_status()
1121 build_dir = build.get_toml('build-dir', 'build') or 'build'
1122 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1124 data = stage0_data(build.rust_root)
1125 build.date = data['date']
1126 build.rustc_channel = data['rustc']
1128 if "rustfmt" in data:
1129 build.rustfmt_channel = data['rustfmt']
1132 build.set_dev_environment()
1134 build.set_normal_environment()
1136 build.build = args.build or build.build_triple()
1137 build.update_submodules()
1139 # Fetch/build the bootstrap
1140 build.rustc_commit = build.maybe_download_rustc()
1141 if build.rustc_commit is not None:
1143 commit = build.rustc_commit
1144 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
1145 # FIXME: support downloading artifacts from the beta channel
1146 build.rustc_channel = "nightly"
1147 build.download_stage0()
1149 build.ensure_vendored()
1150 build.build_bootstrap()
1154 args = [build.bootstrap_binary()]
1155 args.extend(sys.argv[1:])
1156 env = os.environ.copy()
1157 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1158 env["BOOTSTRAP_PYTHON"] = sys.executable
1159 env["BUILD_DIR"] = build.build_dir
1160 env["RUSTC_BOOTSTRAP"] = '1'
1162 env["BOOTSTRAP_CONFIG"] = toml_path
1163 run(args, env=env, verbose=build.verbose)
1167 """Entry point for the bootstrap process"""
1170 # x.py help <cmd> ...
1171 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1172 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1175 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1177 bootstrap(help_triggered)
1178 if not help_triggered:
1179 print("Build completed successfully in {}".format(
1180 format_build_time(time() - start_time)))
1181 except (SystemExit, KeyboardInterrupt) as error:
1182 if hasattr(error, 'code') and isinstance(error.code, int):
1183 exit_code = error.code
1187 if not help_triggered:
1188 print("Build completed unsuccessfully in {}".format(
1189 format_build_time(time() - start_time)))
1193 if __name__ == '__main__':