1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see https://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
154 # For most failures, we definitely do want to print this error, or the user will have no
155 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
156 # have already printed an error above, so there's no need to print the exact command we're
164 def require(cmd, exit=True):
165 '''Run a command, returning its output.
167 If `exit` is `True`, exit the process.
168 Otherwise, return None.'''
170 return subprocess.check_output(cmd).strip()
171 except (subprocess.CalledProcessError, OSError) as exc:
174 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
175 print("Please make sure it's installed and in the path.")
179 def stage0_data(rust_root):
180 """Build a dictionary from stage0.txt"""
181 nightlies = os.path.join(rust_root, "src/stage0.txt")
182 with open(nightlies, 'r') as nightlies:
183 lines = [line.rstrip() for line in nightlies
184 if not line.startswith("#")]
185 return dict([line.split(": ", 1) for line in lines if line])
188 def format_build_time(duration):
189 """Return a nicer format for build time
191 >>> format_build_time('300')
194 return str(datetime.timedelta(seconds=int(duration)))
197 def default_build_triple(verbose):
198 """Build triple as in LLVM"""
199 # If the user already has a host build triple with an existing `rustc`
200 # install, use their preference. This fixes most issues with Windows builds
201 # being detected as GNU instead of MSVC.
202 default_encoding = sys.getdefaultencoding()
204 version = subprocess.check_output(["rustc", "--version", "--verbose"],
205 stderr=subprocess.DEVNULL)
206 version = version.decode(default_encoding)
207 host = next(x for x in version.split('\n') if x.startswith("host: "))
208 triple = host.split("host: ")[1]
210 print("detected default triple {}".format(triple))
212 except Exception as e:
214 print("rustup not detected: {}".format(e))
215 print("falling back to auto-detect")
217 required = sys.platform != 'win32'
218 ostype = require(["uname", "-s"], exit=required)
219 cputype = require(['uname', '-m'], exit=required)
221 # If we do not have `uname`, assume Windows.
222 if ostype is None or cputype is None:
223 return 'x86_64-pc-windows-msvc'
225 ostype = ostype.decode(default_encoding)
226 cputype = cputype.decode(default_encoding)
228 # The goal here is to come up with the same triple as LLVM would,
229 # at least for the subset of platforms we're willing to target.
231 'Darwin': 'apple-darwin',
232 'DragonFly': 'unknown-dragonfly',
233 'FreeBSD': 'unknown-freebsd',
234 'Haiku': 'unknown-haiku',
235 'NetBSD': 'unknown-netbsd',
236 'OpenBSD': 'unknown-openbsd'
239 # Consider the direct transformation first and then the special cases
240 if ostype in ostype_mapper:
241 ostype = ostype_mapper[ostype]
242 elif ostype == 'Linux':
243 os_from_sp = subprocess.check_output(
244 ['uname', '-o']).strip().decode(default_encoding)
245 if os_from_sp == 'Android':
246 ostype = 'linux-android'
248 ostype = 'unknown-linux-gnu'
249 elif ostype == 'SunOS':
250 ostype = 'pc-solaris'
251 # On Solaris, uname -m will return a machine classification instead
252 # of a cpu type, so uname -p is recommended instead. However, the
253 # output from that option is too generic for our purposes (it will
254 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
255 # must be used instead.
256 cputype = require(['isainfo', '-k']).decode(default_encoding)
257 # sparc cpus have sun as a target vendor
258 if 'sparc' in cputype:
259 ostype = 'sun-solaris'
260 elif ostype.startswith('MINGW'):
261 # msys' `uname` does not print gcc configuration, but prints msys
262 # configuration. so we cannot believe `uname -m`:
263 # msys1 is always i686 and msys2 is always x86_64.
264 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
266 ostype = 'pc-windows-gnu'
268 if os.environ.get('MSYSTEM') == 'MINGW64':
270 elif ostype.startswith('MSYS'):
271 ostype = 'pc-windows-gnu'
272 elif ostype.startswith('CYGWIN_NT'):
274 if ostype.endswith('WOW64'):
276 ostype = 'pc-windows-gnu'
277 elif sys.platform == 'win32':
278 # Some Windows platforms might have a `uname` command that returns a
279 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
280 # these cases, fall back to using sys.platform.
281 return 'x86_64-pc-windows-msvc'
283 err = "unknown OS type: {}".format(ostype)
286 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
287 cputype = subprocess.check_output(
288 ['uname', '-p']).strip().decode(default_encoding)
291 'aarch64': 'aarch64',
298 'powerpc': 'powerpc',
299 'powerpc64': 'powerpc64',
300 'powerpc64le': 'powerpc64le',
302 'ppc64': 'powerpc64',
303 'ppc64le': 'powerpc64le',
311 # Consider the direct transformation first and then the special cases
312 if cputype in cputype_mapper:
313 cputype = cputype_mapper[cputype]
314 elif cputype in {'xscale', 'arm'}:
316 if ostype == 'linux-android':
317 ostype = 'linux-androideabi'
318 elif ostype == 'unknown-freebsd':
319 cputype = subprocess.check_output(
320 ['uname', '-p']).strip().decode(default_encoding)
321 ostype = 'unknown-freebsd'
322 elif cputype == 'armv6l':
324 if ostype == 'linux-android':
325 ostype = 'linux-androideabi'
328 elif cputype in {'armv7l', 'armv8l'}:
330 if ostype == 'linux-android':
331 ostype = 'linux-androideabi'
334 elif cputype == 'mips':
335 if sys.byteorder == 'big':
337 elif sys.byteorder == 'little':
340 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
341 elif cputype == 'mips64':
342 if sys.byteorder == 'big':
344 elif sys.byteorder == 'little':
347 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
348 # only the n64 ABI is supported, indicate it
350 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
353 err = "unknown cpu type: {}".format(cputype)
356 return "{}-{}".format(cputype, ostype)
359 @contextlib.contextmanager
360 def output(filepath):
361 tmp = filepath + '.tmp'
362 with open(tmp, 'w') as f:
365 if os.path.exists(filepath):
366 os.remove(filepath) # PermissionError/OSError on Win32 if in use
368 shutil.copy2(tmp, filepath)
371 os.rename(tmp, filepath)
374 class RustBuild(object):
375 """Provide all the methods required to build Rust"""
378 self._download_url = ''
379 self.rustc_channel = ''
380 self.rustfmt_channel = ''
384 self.config_toml = ''
386 self.use_locked_deps = ''
387 self.use_vendored_sources = ''
389 self.git_version = None
390 self.nix_deps_dir = None
391 self.rustc_commit = None
393 def download_toolchain(self, stage0=True, rustc_channel=None):
394 """Fetch the build system for Rust, written in Rust
396 This method will build a cache directory, then it will fetch the
397 tarball which has the stage0 compiler used to then bootstrap the Rust
400 Each downloaded tarball is extracted, after that, the script
401 will move all the content to the right place.
403 if rustc_channel is None:
404 rustc_channel = self.rustc_channel
405 rustfmt_channel = self.rustfmt_channel
406 bin_root = self.bin_root(stage0)
410 key += str(self.rustc_commit)
411 if self.rustc(stage0).startswith(bin_root) and \
412 (not os.path.exists(self.rustc(stage0)) or
413 self.program_out_of_date(self.rustc_stamp(stage0), key)):
414 if os.path.exists(bin_root):
415 shutil.rmtree(bin_root)
416 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
417 filename = "rust-std-{}-{}{}".format(
418 rustc_channel, self.build, tarball_suffix)
419 pattern = "rust-std-{}".format(self.build)
420 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
421 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
423 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
424 # download-rustc doesn't need its own cargo, it can just use beta's.
426 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
428 self._download_component_helper(filename, "cargo", tarball_suffix)
429 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
431 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
432 self._download_component_helper(
433 filename, "rustc-dev", tarball_suffix, stage0
436 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
437 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
438 lib_dir = "{}/lib".format(bin_root)
439 for lib in os.listdir(lib_dir):
440 if lib.endswith(".so"):
441 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
442 with output(self.rustc_stamp(stage0)) as rust_stamp:
443 rust_stamp.write(key)
445 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
446 not os.path.exists(self.rustfmt())
447 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
450 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
451 [channel, date] = rustfmt_channel.split('-', 1)
452 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
453 self._download_component_helper(
454 filename, "rustfmt-preview", tarball_suffix, key=date
456 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
457 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
458 with output(self.rustfmt_stamp()) as rustfmt_stamp:
459 rustfmt_stamp.write(self.rustfmt_channel)
461 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
462 if self.downloading_llvm() and stage0:
463 # We want the most recent LLVM submodule update to avoid downloading
464 # LLVM more often than necessary.
466 # This git command finds that commit SHA, looking for bors-authored
467 # merges that modified src/llvm-project.
469 # This works even in a repository that has not yet initialized
471 top_level = subprocess.check_output([
472 "git", "rev-parse", "--show-toplevel",
473 ]).decode(sys.getdefaultencoding()).strip()
474 llvm_sha = subprocess.check_output([
475 "git", "log", "--author=bors", "--format=%H", "-n1",
476 "-m", "--first-parent",
478 "{}/src/llvm-project".format(top_level),
479 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
480 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
481 "{}/src/version".format(top_level)
482 ]).decode(sys.getdefaultencoding()).strip()
483 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
484 llvm_root = self.llvm_root()
485 llvm_lib = os.path.join(llvm_root, "lib")
486 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
487 self._download_ci_llvm(llvm_sha, llvm_assertions)
488 for binary in ["llvm-config", "FileCheck"]:
489 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
490 for lib in os.listdir(llvm_lib):
491 if lib.endswith(".so"):
492 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
493 with output(self.llvm_stamp()) as llvm_stamp:
494 llvm_stamp.write(llvm_sha + str(llvm_assertions))
496 def downloading_llvm(self):
497 opt = self.get_toml('download-ci-llvm', 'llvm')
498 # This is currently all tier 1 targets (since others may not have CI
500 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
501 supported_platforms = [
502 "aarch64-unknown-linux-gnu",
503 "i686-pc-windows-gnu",
504 "i686-pc-windows-msvc",
505 "i686-unknown-linux-gnu",
506 "x86_64-unknown-linux-gnu",
507 "x86_64-apple-darwin",
508 "x86_64-pc-windows-gnu",
509 "x86_64-pc-windows-msvc",
511 return opt == "true" \
512 or (opt == "if-available" and self.build in supported_platforms)
514 def _download_component_helper(
515 self, filename, pattern, tarball_suffix, stage0=True, key=None
521 key = self.rustc_commit
522 cache_dst = os.path.join(self.build_dir, "cache")
523 rustc_cache = os.path.join(cache_dst, key)
524 if not os.path.exists(rustc_cache):
525 os.makedirs(rustc_cache)
528 url = "{}/dist/{}".format(self._download_url, key)
530 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
531 tarball = os.path.join(rustc_cache, filename)
532 if not os.path.exists(tarball):
533 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
534 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
536 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
537 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
538 cache_dst = os.path.join(self.build_dir, "cache")
539 rustc_cache = os.path.join(cache_dst, cache_prefix)
540 if not os.path.exists(rustc_cache):
541 os.makedirs(rustc_cache)
543 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
545 url = url.replace('rustc-builds', 'rustc-builds-alt')
546 # ci-artifacts are only stored as .xz, not .gz
548 print("error: XZ support is required to download LLVM")
549 print("help: consider disabling `download-ci-llvm` or using python3")
551 tarball_suffix = '.tar.xz'
552 filename = "rust-dev-nightly-" + self.build + tarball_suffix
553 tarball = os.path.join(rustc_cache, filename)
554 if not os.path.exists(tarball):
555 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
556 unpack(tarball, tarball_suffix, self.llvm_root(),
558 verbose=self.verbose)
560 def fix_bin_or_dylib(self, fname):
561 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
562 or the RPATH section, to fix the dynamic library search path
564 This method is only required on NixOS and uses the PatchELF utility to
565 change the interpreter/RPATH of ELF executables.
567 Please see https://nixos.org/patchelf.html for more information
569 default_encoding = sys.getdefaultencoding()
571 ostype = subprocess.check_output(
572 ['uname', '-s']).strip().decode(default_encoding)
573 except subprocess.CalledProcessError:
575 except OSError as reason:
576 if getattr(reason, 'winerror', None) is not None:
580 if ostype != "Linux":
583 if not os.path.exists("/etc/NIXOS"):
585 if os.path.exists("/lib"):
588 # At this point we're pretty sure the user is running NixOS
589 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
590 print(nix_os_msg, fname)
592 # Only build `.nix-deps` once.
593 nix_deps_dir = self.nix_deps_dir
595 # Run `nix-build` to "build" each dependency (which will likely reuse
596 # the existing `/nix/store` copy, or at most download a pre-built copy).
598 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
599 # directory, but still reference the actual `/nix/store` path in the rpath
600 # as it makes it significantly more robust against changes to the location of
601 # the `.nix-deps` location.
603 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
604 # zlib: Needed as a system dependency of `libLLVM-*.so`.
605 # patchelf: Needed for patching ELF binaries (see doc comment above).
606 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
608 with (import <nixpkgs> {});
610 name = "rust-stage0-dependencies";
619 subprocess.check_output([
620 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
622 except subprocess.CalledProcessError as reason:
623 print("warning: failed to call nix-build:", reason)
625 self.nix_deps_dir = nix_deps_dir
627 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
629 # Relative default, all binary and dynamic libraries we ship
630 # appear to have this (even when `../lib` is redundant).
632 os.path.join(os.path.realpath(nix_deps_dir), "lib")
634 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
635 if not fname.endswith(".so"):
636 # Finally, set the corret .interp for binaries
637 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
638 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
641 subprocess.check_output([patchelf] + patchelf_args + [fname])
642 except subprocess.CalledProcessError as reason:
643 print("warning: failed to call patchelf:", reason)
646 # If `download-rustc` is set, download the most recent commit with CI artifacts
647 def maybe_download_ci_toolchain(self):
648 # If `download-rustc` is not set, default to rebuilding.
649 download_rustc = self.get_toml("download-rustc", section="rust")
650 if download_rustc is None or download_rustc == "false":
652 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
654 # Handle running from a directory other than the top level
655 rev_parse = ["git", "rev-parse", "--show-toplevel"]
656 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
657 compiler = "{}/compiler/".format(top_level)
658 library = "{}/library/".format(top_level)
660 # Look for a version to compare to based on the current commit.
661 # Only commits merged by bors will have CI artifacts.
662 merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"]
663 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
665 # Warn if there were changes to the compiler or standard library since the ancestor commit.
666 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
668 if download_rustc == "if-unchanged":
670 print("warning: `download-rustc` is enabled, but there are changes to \
671 compiler/ or library/")
674 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
675 self.rustc_commit = commit
676 # FIXME: support downloading artifacts from the beta channel
677 self.download_toolchain(False, "nightly")
679 def rustc_stamp(self, stage0):
680 """Return the path for .rustc-stamp at the given stage
683 >>> rb.build_dir = "build"
684 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
686 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
689 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
691 def rustfmt_stamp(self):
692 """Return the path for .rustfmt-stamp
695 >>> rb.build_dir = "build"
696 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
699 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
701 def llvm_stamp(self):
702 """Return the path for .rustfmt-stamp
705 >>> rb.build_dir = "build"
706 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
709 return os.path.join(self.llvm_root(), '.llvm-stamp')
712 def program_out_of_date(self, stamp_path, key):
713 """Check if the given program stamp is out of date"""
714 if not os.path.exists(stamp_path) or self.clean:
716 with open(stamp_path, 'r') as stamp:
717 return key != stamp.read()
719 def bin_root(self, stage0):
720 """Return the binary root directory for the given stage
723 >>> rb.build_dir = "build"
724 >>> rb.bin_root(True) == os.path.join("build", "stage0")
726 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
729 When the 'build' property is given should be a nested directory:
731 >>> rb.build = "devel"
732 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
739 return os.path.join(self.build_dir, self.build, subdir)
742 """Return the CI LLVM root directory
745 >>> rb.build_dir = "build"
746 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
749 When the 'build' property is given should be a nested directory:
751 >>> rb.build = "devel"
752 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
755 return os.path.join(self.build_dir, self.build, "ci-llvm")
757 def get_toml(self, key, section=None):
758 """Returns the value of the given key in config.toml, otherwise returns None
761 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
762 >>> rb.get_toml("key2")
765 If the key does not exists, the result is None:
767 >>> rb.get_toml("key3") is None
770 Optionally also matches the section the key appears in
772 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
773 >>> rb.get_toml('key', 'a')
775 >>> rb.get_toml('key', 'b')
777 >>> rb.get_toml('key', 'c') is None
780 >>> rb.config_toml = 'key1 = true'
781 >>> rb.get_toml("key1")
786 for line in self.config_toml.splitlines():
787 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
788 if section_match is not None:
789 cur_section = section_match.group(1)
791 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
792 if match is not None:
793 value = match.group(1)
794 if section is None or section == cur_section:
795 return self.get_string(value) or value.strip()
799 """Return config path for cargo"""
800 return self.program_config('cargo')
802 def rustc(self, stage0):
803 """Return config path for rustc"""
804 return self.program_config('rustc', stage0)
807 """Return config path for rustfmt"""
808 if not self.rustfmt_channel:
810 return self.program_config('rustfmt')
812 def program_config(self, program, stage0=True):
813 """Return config path for the given program at the given stage
816 >>> rb.config_toml = 'rustc = "rustc"\\n'
817 >>> rb.program_config('rustc')
819 >>> rb.config_toml = ''
820 >>> cargo_path = rb.program_config('cargo', True)
821 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
824 >>> cargo_path = rb.program_config('cargo', False)
825 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
829 config = self.get_toml(program)
831 return os.path.expanduser(config)
832 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
833 program, self.exe_suffix()))
836 def get_string(line):
837 """Return the value between double quotes
839 >>> RustBuild.get_string(' "devel" ')
841 >>> RustBuild.get_string(" 'devel' ")
843 >>> RustBuild.get_string('devel') is None
845 >>> RustBuild.get_string(' "devel ')
848 start = line.find('"')
850 end = start + 1 + line[start + 1:].find('"')
851 return line[start + 1:end]
852 start = line.find('\'')
854 end = start + 1 + line[start + 1:].find('\'')
855 return line[start + 1:end]
860 """Return a suffix for executables"""
861 if sys.platform == 'win32':
865 def bootstrap_binary(self):
866 """Return the path of the bootstrap binary
869 >>> rb.build_dir = "build"
870 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
871 ... "debug", "bootstrap")
874 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
876 def build_bootstrap(self):
877 """Build bootstrap"""
878 build_dir = os.path.join(self.build_dir, "bootstrap")
879 if self.clean and os.path.exists(build_dir):
880 shutil.rmtree(build_dir)
881 env = os.environ.copy()
882 # `CARGO_BUILD_TARGET` breaks bootstrap build.
883 # See also: <https://github.com/rust-lang/rust/issues/70208>.
884 if "CARGO_BUILD_TARGET" in env:
885 del env["CARGO_BUILD_TARGET"]
886 env["CARGO_TARGET_DIR"] = build_dir
887 env["RUSTC"] = self.rustc(True)
888 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
889 (os.pathsep + env["LD_LIBRARY_PATH"]) \
890 if "LD_LIBRARY_PATH" in env else ""
891 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
892 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
893 if "DYLD_LIBRARY_PATH" in env else ""
894 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
895 (os.pathsep + env["LIBRARY_PATH"]) \
896 if "LIBRARY_PATH" in env else ""
897 # preserve existing RUSTFLAGS
898 env.setdefault("RUSTFLAGS", "")
899 env["RUSTFLAGS"] += " -Cdebuginfo=2"
901 build_section = "target.{}".format(self.build)
903 if self.get_toml("crt-static", build_section) == "true":
904 target_features += ["+crt-static"]
905 elif self.get_toml("crt-static", build_section) == "false":
906 target_features += ["-crt-static"]
908 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
909 target_linker = self.get_toml("linker", build_section)
910 if target_linker is not None:
911 env["RUSTFLAGS"] += " -C linker=" + target_linker
912 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
913 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
914 if self.get_toml("deny-warnings", "rust") != "false":
915 env["RUSTFLAGS"] += " -Dwarnings"
917 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
918 os.pathsep + env["PATH"]
919 if not os.path.isfile(self.cargo()):
920 raise Exception("no cargo executable found at `{}`".format(
922 args = [self.cargo(), "build", "--manifest-path",
923 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
924 for _ in range(1, self.verbose):
925 args.append("--verbose")
926 if self.use_locked_deps:
927 args.append("--locked")
928 if self.use_vendored_sources:
929 args.append("--frozen")
930 run(args, env=env, verbose=self.verbose)
932 def build_triple(self):
933 """Build triple as in LLVM
935 Note that `default_build_triple` is moderately expensive,
936 so use `self.build` where possible.
938 config = self.get_toml('build')
941 return default_build_triple(self.verbose)
943 def check_submodule(self, module, slow_submodules):
944 if not slow_submodules:
945 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
946 cwd=os.path.join(self.rust_root, module),
947 stdout=subprocess.PIPE)
952 def update_submodule(self, module, checked_out, recorded_submodules):
953 module_path = os.path.join(self.rust_root, module)
955 if checked_out is not None:
956 default_encoding = sys.getdefaultencoding()
957 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
958 if recorded_submodules[module] == checked_out:
961 print("Updating submodule", module)
963 run(["git", "submodule", "-q", "sync", module],
964 cwd=self.rust_root, verbose=self.verbose)
966 update_args = ["git", "submodule", "update", "--init", "--recursive"]
967 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
968 update_args.append("--progress")
969 update_args.append(module)
970 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
972 run(["git", "reset", "-q", "--hard"],
973 cwd=module_path, verbose=self.verbose)
974 run(["git", "clean", "-qdfx"],
975 cwd=module_path, verbose=self.verbose)
977 def update_submodules(self):
978 """Update submodules"""
979 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
980 self.get_toml('submodules') == "false":
983 default_encoding = sys.getdefaultencoding()
985 # check the existence and version of 'git' command
986 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
987 self.git_version = distutils.version.LooseVersion(git_version_str)
989 slow_submodules = self.get_toml('fast-submodules') == "false"
992 print('Unconditionally updating all submodules')
994 print('Updating only changed submodules')
995 default_encoding = sys.getdefaultencoding()
996 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
997 ["git", "config", "--file",
998 os.path.join(self.rust_root, ".gitmodules"),
999 "--get-regexp", "path"]
1000 ).decode(default_encoding).splitlines()]
1001 filtered_submodules = []
1002 submodules_names = []
1003 for module in submodules:
1004 # This is handled by native::Llvm in rustbuild, not here
1005 if module.endswith("llvm-project"):
1007 check = self.check_submodule(module, slow_submodules)
1008 filtered_submodules.append((module, check))
1009 submodules_names.append(module)
1010 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1011 cwd=self.rust_root, stdout=subprocess.PIPE)
1012 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1013 # { filename: hash }
1014 recorded_submodules = {}
1015 for data in recorded:
1016 # [mode, kind, hash, filename]
1018 recorded_submodules[data[3]] = data[2]
1019 for module in filtered_submodules:
1020 self.update_submodule(module[0], module[1], recorded_submodules)
1021 print("Submodules updated in %.2f seconds" % (time() - start_time))
1023 def set_normal_environment(self):
1024 """Set download URL for normal environment"""
1025 if 'RUSTUP_DIST_SERVER' in os.environ:
1026 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1028 self._download_url = 'https://static.rust-lang.org'
1030 def set_dev_environment(self):
1031 """Set download URL for development environment"""
1032 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1033 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1035 self._download_url = 'https://dev-static.rust-lang.org'
1037 def check_vendored_status(self):
1038 """Check that vendoring is configured properly"""
1039 vendor_dir = os.path.join(self.rust_root, 'vendor')
1040 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1041 if os.environ.get('USER') != os.environ['SUDO_USER']:
1042 self.use_vendored_sources = True
1043 print('info: looks like you are running this command under `sudo`')
1044 print(' and so in order to preserve your $HOME this will now')
1045 print(' use vendored sources by default.')
1046 if not os.path.exists(vendor_dir):
1047 print('error: vendoring required, but vendor directory does not exist.')
1048 print(' Run `cargo vendor` without sudo to initialize the '
1049 'vendor directory.')
1050 raise Exception("{} not found".format(vendor_dir))
1052 if self.use_vendored_sources:
1053 if not os.path.exists('.cargo'):
1054 os.makedirs('.cargo')
1055 with output('.cargo/config') as cargo_config:
1057 "[source.crates-io]\n"
1058 "replace-with = 'vendored-sources'\n"
1059 "registry = 'https://example.com'\n"
1061 "[source.vendored-sources]\n"
1062 "directory = '{}/vendor'\n"
1063 .format(self.rust_root))
1065 if os.path.exists('.cargo'):
1066 shutil.rmtree('.cargo')
1068 def ensure_vendored(self):
1069 """Ensure that the vendored sources are available if needed"""
1070 vendor_dir = os.path.join(self.rust_root, 'vendor')
1071 # Note that this does not handle updating the vendored dependencies if
1072 # the rust git repository is updated. Normal development usually does
1073 # not use vendoring, so hopefully this isn't too much of a problem.
1074 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1078 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1079 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1080 ], verbose=self.verbose, cwd=self.rust_root)
1083 def bootstrap(help_triggered):
1084 """Configure, fetch, build and run the initial bootstrap"""
1086 # If the user is asking for help, let them know that the whole download-and-build
1087 # process has to happen before anything is printed out.
1089 print("info: Downloading and building bootstrap before processing --help")
1090 print(" command. See src/bootstrap/README.md for help with common")
1093 parser = argparse.ArgumentParser(description='Build rust')
1094 parser.add_argument('--config')
1095 parser.add_argument('--build')
1096 parser.add_argument('--clean', action='store_true')
1097 parser.add_argument('-v', '--verbose', action='count', default=0)
1099 args = [a for a in sys.argv if a != '-h' and a != '--help']
1100 args, _ = parser.parse_known_args(args)
1102 # Configure initial bootstrap
1104 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1105 build.verbose = args.verbose
1106 build.clean = args.clean
1108 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1110 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1111 if not toml_path and os.path.exists('config.toml'):
1112 toml_path = 'config.toml'
1115 if not os.path.exists(toml_path):
1116 toml_path = os.path.join(build.rust_root, toml_path)
1118 with open(toml_path) as config:
1119 build.config_toml = config.read()
1121 profile = build.get_toml('profile')
1122 if profile is not None:
1123 include_file = 'config.{}.toml'.format(profile)
1124 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1125 include_path = os.path.join(include_dir, include_file)
1126 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1127 # specific key, so appending our defaults at the end allows the user to override them
1128 with open(include_path) as included_toml:
1129 build.config_toml += os.linesep + included_toml.read()
1131 config_verbose = build.get_toml('verbose', 'build')
1132 if config_verbose is not None:
1133 build.verbose = max(build.verbose, int(config_verbose))
1135 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1137 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1139 build.check_vendored_status()
1141 build_dir = build.get_toml('build-dir', 'build') or 'build'
1142 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1144 data = stage0_data(build.rust_root)
1145 build.date = data['date']
1146 build.rustc_channel = data['rustc']
1148 if "rustfmt" in data:
1149 build.rustfmt_channel = data['rustfmt']
1152 build.set_dev_environment()
1154 build.set_normal_environment()
1156 build.build = args.build or build.build_triple()
1157 build.update_submodules()
1159 # Fetch/build the bootstrap
1160 build.download_toolchain()
1161 # Download the master compiler if `download-rustc` is set
1162 build.maybe_download_ci_toolchain()
1164 build.ensure_vendored()
1165 build.build_bootstrap()
1169 args = [build.bootstrap_binary()]
1170 args.extend(sys.argv[1:])
1171 env = os.environ.copy()
1172 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1173 env["BOOTSTRAP_PYTHON"] = sys.executable
1174 env["BUILD_DIR"] = build.build_dir
1175 env["RUSTC_BOOTSTRAP"] = '1'
1177 env["BOOTSTRAP_CONFIG"] = toml_path
1178 if build.rustc_commit is not None:
1179 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1180 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1184 """Entry point for the bootstrap process"""
1187 # x.py help <cmd> ...
1188 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1189 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1192 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1194 bootstrap(help_triggered)
1195 if not help_triggered:
1196 print("Build completed successfully in {}".format(
1197 format_build_time(time() - start_time)))
1198 except (SystemExit, KeyboardInterrupt) as error:
1199 if hasattr(error, 'code') and isinstance(error.code, int):
1200 exit_code = error.code
1204 if not help_triggered:
1205 print("Build completed unsuccessfully in {}".format(
1206 format_build_time(time() - start_time)))
1210 if __name__ == '__main__':