1 from __future__ import absolute_import, division, print_function
5 import distutils.version
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
25 path = os.path.join(build_dir, "lock.db")
27 con = sqlite3.Connection(path, timeout=0)
29 curs.execute("BEGIN EXCLUSIVE")
30 # The lock is released when the cursor is dropped
32 # If the database is busy then lock has already been acquired
33 # so we wait for the lock.
34 # We retry every quarter second so that execution is passed back to python
35 # so that it can handle signals
36 except sqlite3.OperationalError:
39 print("Waiting for lock on build directory")
40 con = sqlite3.Connection(path, timeout=0.25)
44 curs.execute("BEGIN EXCLUSIVE")
45 except sqlite3.OperationalError:
49 print("warning: sqlite3 not available in python, skipping build directory lock")
50 print("please file an issue on rust-lang/rust")
51 print("this is not a problem for non-concurrent x.py invocations")
56 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
57 temp_path = temp_file.name
58 with tarfile.open(temp_path, "w:xz"):
61 except tarfile.CompressionError:
64 def get(base, url, path, checksums, verbose=False, do_verify=True):
65 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
66 temp_path = temp_file.name
70 if url not in checksums:
71 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
72 sha256 = checksums[url]
73 if os.path.exists(path):
74 if verify(path, sha256, False):
76 print("using already-download file", path)
80 print("ignoring already-download file",
81 path, "due to failed verification")
83 download(temp_path, "{}/{}".format(base, url), True, verbose)
84 if do_verify and not verify(temp_path, sha256, verbose):
85 raise RuntimeError("failed verification")
87 print("moving {} to {}".format(temp_path, path))
88 shutil.move(temp_path, path)
90 if os.path.isfile(temp_path):
92 print("removing", temp_path)
96 def download(path, url, probably_big, verbose):
99 _download(path, url, probably_big, verbose, True)
102 print("\nspurious failure, trying again")
103 _download(path, url, probably_big, verbose, False)
106 def _download(path, url, probably_big, verbose, exception):
107 if probably_big or verbose:
108 print("downloading {}".format(url))
109 # see https://serverfault.com/questions/301128/how-to-download
110 if sys.platform == 'win32':
111 run(["PowerShell.exe", "/nologo", "-Command",
112 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
113 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
117 if probably_big or verbose:
121 require(["curl", "--version"])
123 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
124 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
125 "--retry", "3", "-Sf", "-o", path, url],
130 def verify(path, expected, verbose):
131 """Check if the sha256 sum of the given path is valid"""
133 print("verifying", path)
134 with open(path, "rb") as source:
135 found = hashlib.sha256(source.read()).hexdigest()
136 verified = found == expected
138 print("invalid checksum:\n"
140 " expected: {}".format(found, expected))
144 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
145 """Unpack the given tarball file"""
146 print("extracting", tarball)
147 fname = os.path.basename(tarball).replace(tarball_suffix, "")
148 with contextlib.closing(tarfile.open(tarball)) as tar:
149 for member in tar.getnames():
150 if "/" not in member:
152 name = member.replace(fname + "/", "", 1)
153 if match is not None and not name.startswith(match):
155 name = name[len(match) + 1:]
157 dst_path = os.path.join(dst, name)
159 print(" extracting", member)
160 tar.extract(member, dst)
161 src_path = os.path.join(dst, member)
162 if os.path.isdir(src_path) and os.path.exists(dst_path):
164 shutil.move(src_path, dst_path)
165 shutil.rmtree(os.path.join(dst, fname))
168 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
169 """Run a child program in a new process"""
171 print("running: " + ' '.join(args))
173 # Use Popen here instead of call() as it apparently allows powershell on
174 # Windows to not lock up waiting for input presumably.
175 ret = subprocess.Popen(args, **kwargs)
178 err = "failed to run: " + ' '.join(args)
179 if verbose or exception:
180 raise RuntimeError(err)
181 # For most failures, we definitely do want to print this error, or the user will have no
182 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
183 # have already printed an error above, so there's no need to print the exact command we're
191 def require(cmd, exit=True):
192 '''Run a command, returning its output.
194 If `exit` is `True`, exit the process.
195 Otherwise, return None.'''
197 return subprocess.check_output(cmd).strip()
198 except (subprocess.CalledProcessError, OSError) as exc:
201 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
202 print("Please make sure it's installed and in the path.")
206 def format_build_time(duration):
207 """Return a nicer format for build time
209 >>> format_build_time('300')
212 return str(datetime.timedelta(seconds=int(duration)))
215 def default_build_triple(verbose):
216 """Build triple as in LLVM"""
217 # If the user already has a host build triple with an existing `rustc`
218 # install, use their preference. This fixes most issues with Windows builds
219 # being detected as GNU instead of MSVC.
220 default_encoding = sys.getdefaultencoding()
222 version = subprocess.check_output(["rustc", "--version", "--verbose"],
223 stderr=subprocess.DEVNULL)
224 version = version.decode(default_encoding)
225 host = next(x for x in version.split('\n') if x.startswith("host: "))
226 triple = host.split("host: ")[1]
228 print("detected default triple {} from pre-installed rustc".format(triple))
230 except Exception as e:
232 print("pre-installed rustc not detected: {}".format(e))
233 print("falling back to auto-detect")
235 required = sys.platform != 'win32'
236 ostype = require(["uname", "-s"], exit=required)
237 cputype = require(['uname', '-m'], exit=required)
239 # If we do not have `uname`, assume Windows.
240 if ostype is None or cputype is None:
241 return 'x86_64-pc-windows-msvc'
243 ostype = ostype.decode(default_encoding)
244 cputype = cputype.decode(default_encoding)
246 # The goal here is to come up with the same triple as LLVM would,
247 # at least for the subset of platforms we're willing to target.
249 'Darwin': 'apple-darwin',
250 'DragonFly': 'unknown-dragonfly',
251 'FreeBSD': 'unknown-freebsd',
252 'Haiku': 'unknown-haiku',
253 'NetBSD': 'unknown-netbsd',
254 'OpenBSD': 'unknown-openbsd'
257 # Consider the direct transformation first and then the special cases
258 if ostype in ostype_mapper:
259 ostype = ostype_mapper[ostype]
260 elif ostype == 'Linux':
261 os_from_sp = subprocess.check_output(
262 ['uname', '-o']).strip().decode(default_encoding)
263 if os_from_sp == 'Android':
264 ostype = 'linux-android'
266 ostype = 'unknown-linux-gnu'
267 elif ostype == 'SunOS':
268 ostype = 'pc-solaris'
269 # On Solaris, uname -m will return a machine classification instead
270 # of a cpu type, so uname -p is recommended instead. However, the
271 # output from that option is too generic for our purposes (it will
272 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
273 # must be used instead.
274 cputype = require(['isainfo', '-k']).decode(default_encoding)
275 # sparc cpus have sun as a target vendor
276 if 'sparc' in cputype:
277 ostype = 'sun-solaris'
278 elif ostype.startswith('MINGW'):
279 # msys' `uname` does not print gcc configuration, but prints msys
280 # configuration. so we cannot believe `uname -m`:
281 # msys1 is always i686 and msys2 is always x86_64.
282 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
284 ostype = 'pc-windows-gnu'
286 if os.environ.get('MSYSTEM') == 'MINGW64':
288 elif ostype.startswith('MSYS'):
289 ostype = 'pc-windows-gnu'
290 elif ostype.startswith('CYGWIN_NT'):
292 if ostype.endswith('WOW64'):
294 ostype = 'pc-windows-gnu'
295 elif sys.platform == 'win32':
296 # Some Windows platforms might have a `uname` command that returns a
297 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
298 # these cases, fall back to using sys.platform.
299 return 'x86_64-pc-windows-msvc'
301 err = "unknown OS type: {}".format(ostype)
304 if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
305 cputype = subprocess.check_output(
306 ['uname', '-p']).strip().decode(default_encoding)
309 'aarch64': 'aarch64',
317 'powerpc': 'powerpc',
318 'powerpc64': 'powerpc64',
319 'powerpc64le': 'powerpc64le',
321 'ppc64': 'powerpc64',
322 'ppc64le': 'powerpc64le',
323 'riscv64': 'riscv64gc',
331 # Consider the direct transformation first and then the special cases
332 if cputype in cputype_mapper:
333 cputype = cputype_mapper[cputype]
334 elif cputype in {'xscale', 'arm'}:
336 if ostype == 'linux-android':
337 ostype = 'linux-androideabi'
338 elif ostype == 'unknown-freebsd':
339 cputype = subprocess.check_output(
340 ['uname', '-p']).strip().decode(default_encoding)
341 ostype = 'unknown-freebsd'
342 elif cputype == 'armv6l':
344 if ostype == 'linux-android':
345 ostype = 'linux-androideabi'
348 elif cputype in {'armv7l', 'armv8l'}:
350 if ostype == 'linux-android':
351 ostype = 'linux-androideabi'
354 elif cputype == 'mips':
355 if sys.byteorder == 'big':
357 elif sys.byteorder == 'little':
360 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
361 elif cputype == 'mips64':
362 if sys.byteorder == 'big':
364 elif sys.byteorder == 'little':
367 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
368 # only the n64 ABI is supported, indicate it
370 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
373 err = "unknown cpu type: {}".format(cputype)
376 return "{}-{}".format(cputype, ostype)
379 @contextlib.contextmanager
380 def output(filepath):
381 tmp = filepath + '.tmp'
382 with open(tmp, 'w') as f:
385 if os.path.exists(filepath):
386 os.remove(filepath) # PermissionError/OSError on Win32 if in use
388 shutil.copy2(tmp, filepath)
391 os.rename(tmp, filepath)
394 class Stage0Toolchain:
395 def __init__(self, stage0_payload):
396 self.date = stage0_payload["date"]
397 self.version = stage0_payload["version"]
400 return self.version + "-" + self.date
403 class RustBuild(object):
404 """Provide all the methods required to build Rust"""
406 self.checksums_sha256 = {}
407 self.stage0_compiler = None
408 self.stage0_rustfmt = None
409 self._download_url = ''
413 self.config_toml = ''
415 self.use_locked_deps = ''
416 self.use_vendored_sources = ''
418 self.git_version = None
419 self.nix_deps_dir = None
420 self.rustc_commit = None
422 def download_toolchain(self, stage0=True, rustc_channel=None):
423 """Fetch the build system for Rust, written in Rust
425 This method will build a cache directory, then it will fetch the
426 tarball which has the stage0 compiler used to then bootstrap the Rust
429 Each downloaded tarball is extracted, after that, the script
430 will move all the content to the right place.
432 if rustc_channel is None:
433 rustc_channel = self.stage0_compiler.version
434 bin_root = self.bin_root(stage0)
436 key = self.stage0_compiler.date
438 key += str(self.rustc_commit)
439 if self.rustc(stage0).startswith(bin_root) and \
440 (not os.path.exists(self.rustc(stage0)) or
441 self.program_out_of_date(self.rustc_stamp(stage0), key)):
442 if os.path.exists(bin_root):
443 shutil.rmtree(bin_root)
444 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
445 filename = "rust-std-{}-{}{}".format(
446 rustc_channel, self.build, tarball_suffix)
447 pattern = "rust-std-{}".format(self.build)
448 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
449 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
451 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
452 # download-rustc doesn't need its own cargo, it can just use beta's.
454 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
456 self._download_component_helper(filename, "cargo", tarball_suffix)
457 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
459 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
460 self._download_component_helper(
461 filename, "rustc-dev", tarball_suffix, stage0
464 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
465 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
466 lib_dir = "{}/lib".format(bin_root)
467 for lib in os.listdir(lib_dir):
468 if lib.endswith(".so"):
469 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
470 with output(self.rustc_stamp(stage0)) as rust_stamp:
471 rust_stamp.write(key)
473 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
474 not os.path.exists(self.rustfmt())
475 or self.program_out_of_date(
476 self.rustfmt_stamp(),
477 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
480 if self.stage0_rustfmt is not None:
481 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
482 filename = "rustfmt-{}-{}{}".format(
483 self.stage0_rustfmt.version, self.build, tarball_suffix,
485 self._download_component_helper(
486 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
488 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
489 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
490 with output(self.rustfmt_stamp()) as rustfmt_stamp:
491 rustfmt_stamp.write(self.stage0_rustfmt.channel())
493 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
494 if self.downloading_llvm() and stage0:
495 # We want the most recent LLVM submodule update to avoid downloading
496 # LLVM more often than necessary.
498 # This git command finds that commit SHA, looking for bors-authored
499 # commits that modified src/llvm-project or other relevant version
502 # This works even in a repository that has not yet initialized
504 top_level = subprocess.check_output([
505 "git", "rev-parse", "--show-toplevel",
506 ]).decode(sys.getdefaultencoding()).strip()
507 llvm_sha = subprocess.check_output([
508 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
509 "--first-parent", "HEAD",
511 "{}/src/llvm-project".format(top_level),
512 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
513 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
514 "{}/src/version".format(top_level)
515 ]).decode(sys.getdefaultencoding()).strip()
516 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
517 llvm_root = self.llvm_root()
518 llvm_lib = os.path.join(llvm_root, "lib")
519 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
520 self._download_ci_llvm(llvm_sha, llvm_assertions)
521 for binary in ["llvm-config", "FileCheck"]:
522 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
523 for lib in os.listdir(llvm_lib):
524 if lib.endswith(".so"):
525 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
526 with output(self.llvm_stamp()) as llvm_stamp:
527 llvm_stamp.write(llvm_sha + str(llvm_assertions))
529 def downloading_llvm(self):
530 opt = self.get_toml('download-ci-llvm', 'llvm')
531 # This is currently all tier 1 targets and tier 2 targets with host tools
532 # (since others may not have CI artifacts)
533 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
534 supported_platforms = [
536 "aarch64-unknown-linux-gnu",
537 "i686-pc-windows-gnu",
538 "i686-pc-windows-msvc",
539 "i686-unknown-linux-gnu",
540 "x86_64-unknown-linux-gnu",
541 "x86_64-apple-darwin",
542 "x86_64-pc-windows-gnu",
543 "x86_64-pc-windows-msvc",
544 # tier 2 with host tools
545 "aarch64-apple-darwin",
546 "aarch64-pc-windows-msvc",
547 "aarch64-unknown-linux-musl",
548 "arm-unknown-linux-gnueabi",
549 "arm-unknown-linux-gnueabihf",
550 "armv7-unknown-linux-gnueabihf",
551 "mips-unknown-linux-gnu",
552 "mips64-unknown-linux-gnuabi64",
553 "mips64el-unknown-linux-gnuabi64",
554 "mipsel-unknown-linux-gnu",
555 "powerpc-unknown-linux-gnu",
556 "powerpc64-unknown-linux-gnu",
557 "powerpc64le-unknown-linux-gnu",
558 "riscv64gc-unknown-linux-gnu",
559 "s390x-unknown-linux-gnu",
560 "x86_64-unknown-freebsd",
561 "x86_64-unknown-illumos",
562 "x86_64-unknown-linux-musl",
563 "x86_64-unknown-netbsd",
565 return opt == "true" \
566 or (opt == "if-available" and self.build in supported_platforms)
568 def _download_component_helper(
569 self, filename, pattern, tarball_suffix, stage0=True, key=None
573 key = self.stage0_compiler.date
575 key = self.rustc_commit
576 cache_dst = os.path.join(self.build_dir, "cache")
577 rustc_cache = os.path.join(cache_dst, key)
578 if not os.path.exists(rustc_cache):
579 os.makedirs(rustc_cache)
582 base = self._download_url
583 url = "dist/{}".format(key)
585 base = "https://ci-artifacts.rust-lang.org"
586 url = "rustc-builds/{}".format(self.rustc_commit)
587 tarball = os.path.join(rustc_cache, filename)
588 if not os.path.exists(tarball):
591 "{}/{}".format(url, filename),
593 self.checksums_sha256,
594 verbose=self.verbose,
597 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
599 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
601 print("error: could not find commit hash for downloading LLVM")
602 print("help: maybe your repository history is too shallow?")
603 print("help: consider disabling `download-ci-llvm`")
604 print("help: or fetch enough history to include one upstream commit")
606 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
607 cache_dst = os.path.join(self.build_dir, "cache")
608 rustc_cache = os.path.join(cache_dst, cache_prefix)
609 if not os.path.exists(rustc_cache):
610 os.makedirs(rustc_cache)
612 base = "https://ci-artifacts.rust-lang.org"
613 url = "rustc-builds/{}".format(llvm_sha)
615 url = url.replace('rustc-builds', 'rustc-builds-alt')
616 # ci-artifacts are only stored as .xz, not .gz
618 print("error: XZ support is required to download LLVM")
619 print("help: consider disabling `download-ci-llvm` or using python3")
621 tarball_suffix = '.tar.xz'
622 filename = "rust-dev-nightly-" + self.build + tarball_suffix
623 tarball = os.path.join(rustc_cache, filename)
624 if not os.path.exists(tarball):
627 "{}/{}".format(url, filename),
629 self.checksums_sha256,
630 verbose=self.verbose,
633 unpack(tarball, tarball_suffix, self.llvm_root(),
635 verbose=self.verbose)
637 def fix_bin_or_dylib(self, fname):
638 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
639 or the RPATH section, to fix the dynamic library search path
641 This method is only required on NixOS and uses the PatchELF utility to
642 change the interpreter/RPATH of ELF executables.
644 Please see https://nixos.org/patchelf.html for more information
646 default_encoding = sys.getdefaultencoding()
648 ostype = subprocess.check_output(
649 ['uname', '-s']).strip().decode(default_encoding)
650 except subprocess.CalledProcessError:
652 except OSError as reason:
653 if getattr(reason, 'winerror', None) is not None:
657 if ostype != "Linux":
660 # If the user has asked binaries to be patched for Nix, then
661 # don't check for NixOS or `/lib`, just continue to the patching.
662 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
663 # Use `/etc/os-release` instead of `/etc/NIXOS`.
664 # The latter one does not exist on NixOS when using tmpfs as root.
666 with open("/etc/os-release", "r") as f:
667 if not any(line.strip() == "ID=nixos" for line in f):
669 except FileNotFoundError:
671 if os.path.exists("/lib"):
674 # At this point we're pretty sure the user is running NixOS or
676 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
677 print(nix_os_msg, fname)
679 # Only build `.nix-deps` once.
680 nix_deps_dir = self.nix_deps_dir
682 # Run `nix-build` to "build" each dependency (which will likely reuse
683 # the existing `/nix/store` copy, or at most download a pre-built copy).
685 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
686 # directory, but still reference the actual `/nix/store` path in the rpath
687 # as it makes it significantly more robust against changes to the location of
688 # the `.nix-deps` location.
690 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
691 # zlib: Needed as a system dependency of `libLLVM-*.so`.
692 # patchelf: Needed for patching ELF binaries (see doc comment above).
693 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
695 with (import <nixpkgs> {});
697 name = "rust-stage0-dependencies";
706 subprocess.check_output([
707 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
709 except subprocess.CalledProcessError as reason:
710 print("warning: failed to call nix-build:", reason)
712 self.nix_deps_dir = nix_deps_dir
714 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
716 # Relative default, all binary and dynamic libraries we ship
717 # appear to have this (even when `../lib` is redundant).
719 os.path.join(os.path.realpath(nix_deps_dir), "lib")
721 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
722 if not fname.endswith(".so"):
723 # Finally, set the corret .interp for binaries
724 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
725 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
728 subprocess.check_output([patchelf] + patchelf_args + [fname])
729 except subprocess.CalledProcessError as reason:
730 print("warning: failed to call patchelf:", reason)
733 # If `download-rustc` is set, download the most recent commit with CI artifacts
734 def maybe_download_ci_toolchain(self):
735 # If `download-rustc` is not set, default to rebuilding.
736 download_rustc = self.get_toml("download-rustc", section="rust")
737 if download_rustc is None or download_rustc == "false":
739 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
741 # Handle running from a directory other than the top level
742 rev_parse = ["git", "rev-parse", "--show-toplevel"]
743 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
744 compiler = "{}/compiler/".format(top_level)
745 library = "{}/library/".format(top_level)
747 # Look for a version to compare to based on the current commit.
748 # Only commits merged by bors will have CI artifacts.
750 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
751 "--first-parent", "HEAD"
753 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
755 print("error: could not find commit hash for downloading rustc")
756 print("help: maybe your repository history is too shallow?")
757 print("help: consider disabling `download-rustc`")
758 print("help: or fetch enough history to include one upstream commit")
761 # Warn if there were changes to the compiler or standard library since the ancestor commit.
762 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
764 if download_rustc == "if-unchanged":
766 print("warning: saw changes to compiler/ or library/ since {}; " \
767 "ignoring `download-rustc`".format(commit))
769 print("warning: `download-rustc` is enabled, but there are changes to " \
770 "compiler/ or library/")
773 print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
774 self.rustc_commit = commit
775 # FIXME: support downloading artifacts from the beta channel
776 self.download_toolchain(False, "nightly")
778 def rustc_stamp(self, stage0):
779 """Return the path for .rustc-stamp at the given stage
782 >>> rb.build_dir = "build"
783 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
785 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
788 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
790 def rustfmt_stamp(self):
791 """Return the path for .rustfmt-stamp
794 >>> rb.build_dir = "build"
795 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
798 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
800 def llvm_stamp(self):
801 """Return the path for .rustfmt-stamp
804 >>> rb.build_dir = "build"
805 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
808 return os.path.join(self.llvm_root(), '.llvm-stamp')
811 def program_out_of_date(self, stamp_path, key):
812 """Check if the given program stamp is out of date"""
813 if not os.path.exists(stamp_path) or self.clean:
815 with open(stamp_path, 'r') as stamp:
816 return key != stamp.read()
818 def bin_root(self, stage0):
819 """Return the binary root directory for the given stage
822 >>> rb.build_dir = "build"
823 >>> rb.bin_root(True) == os.path.join("build", "stage0")
825 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
828 When the 'build' property is given should be a nested directory:
830 >>> rb.build = "devel"
831 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
838 return os.path.join(self.build_dir, self.build, subdir)
841 """Return the CI LLVM root directory
844 >>> rb.build_dir = "build"
845 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
848 When the 'build' property is given should be a nested directory:
850 >>> rb.build = "devel"
851 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
854 return os.path.join(self.build_dir, self.build, "ci-llvm")
856 def get_toml(self, key, section=None):
857 """Returns the value of the given key in config.toml, otherwise returns None
860 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
861 >>> rb.get_toml("key2")
864 If the key does not exists, the result is None:
866 >>> rb.get_toml("key3") is None
869 Optionally also matches the section the key appears in
871 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
872 >>> rb.get_toml('key', 'a')
874 >>> rb.get_toml('key', 'b')
876 >>> rb.get_toml('key', 'c') is None
879 >>> rb.config_toml = 'key1 = true'
880 >>> rb.get_toml("key1")
885 for line in self.config_toml.splitlines():
886 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
887 if section_match is not None:
888 cur_section = section_match.group(1)
890 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
891 if match is not None:
892 value = match.group(1)
893 if section is None or section == cur_section:
894 return self.get_string(value) or value.strip()
898 """Return config path for cargo"""
899 return self.program_config('cargo')
901 def rustc(self, stage0):
902 """Return config path for rustc"""
903 return self.program_config('rustc', stage0)
906 """Return config path for rustfmt"""
907 if self.stage0_rustfmt is None:
909 return self.program_config('rustfmt')
911 def program_config(self, program, stage0=True):
912 """Return config path for the given program at the given stage
915 >>> rb.config_toml = 'rustc = "rustc"\\n'
916 >>> rb.program_config('rustc')
918 >>> rb.config_toml = ''
919 >>> cargo_path = rb.program_config('cargo', True)
920 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
923 >>> cargo_path = rb.program_config('cargo', False)
924 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
928 config = self.get_toml(program)
930 return os.path.expanduser(config)
931 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
932 program, self.exe_suffix()))
935 def get_string(line):
936 """Return the value between double quotes
938 >>> RustBuild.get_string(' "devel" ')
940 >>> RustBuild.get_string(" 'devel' ")
942 >>> RustBuild.get_string('devel') is None
944 >>> RustBuild.get_string(' "devel ')
947 start = line.find('"')
949 end = start + 1 + line[start + 1:].find('"')
950 return line[start + 1:end]
951 start = line.find('\'')
953 end = start + 1 + line[start + 1:].find('\'')
954 return line[start + 1:end]
959 """Return a suffix for executables"""
960 if sys.platform == 'win32':
964 def bootstrap_binary(self):
965 """Return the path of the bootstrap binary
968 >>> rb.build_dir = "build"
969 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
970 ... "debug", "bootstrap")
973 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
975 def build_bootstrap(self):
976 """Build bootstrap"""
977 build_dir = os.path.join(self.build_dir, "bootstrap")
978 if self.clean and os.path.exists(build_dir):
979 shutil.rmtree(build_dir)
980 env = os.environ.copy()
981 # `CARGO_BUILD_TARGET` breaks bootstrap build.
982 # See also: <https://github.com/rust-lang/rust/issues/70208>.
983 if "CARGO_BUILD_TARGET" in env:
984 del env["CARGO_BUILD_TARGET"]
985 env["CARGO_TARGET_DIR"] = build_dir
986 env["RUSTC"] = self.rustc(True)
987 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
988 (os.pathsep + env["LD_LIBRARY_PATH"]) \
989 if "LD_LIBRARY_PATH" in env else ""
990 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
991 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
992 if "DYLD_LIBRARY_PATH" in env else ""
993 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
994 (os.pathsep + env["LIBRARY_PATH"]) \
995 if "LIBRARY_PATH" in env else ""
997 # preserve existing RUSTFLAGS
998 env.setdefault("RUSTFLAGS", "")
999 build_section = "target.{}".format(self.build)
1000 target_features = []
1001 if self.get_toml("crt-static", build_section) == "true":
1002 target_features += ["+crt-static"]
1003 elif self.get_toml("crt-static", build_section) == "false":
1004 target_features += ["-crt-static"]
1006 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1007 target_linker = self.get_toml("linker", build_section)
1008 if target_linker is not None:
1009 env["RUSTFLAGS"] += " -C linker=" + target_linker
1010 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1011 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1012 if self.get_toml("deny-warnings", "rust") != "false":
1013 env["RUSTFLAGS"] += " -Dwarnings"
1015 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1016 os.pathsep + env["PATH"]
1017 if not os.path.isfile(self.cargo()):
1018 raise Exception("no cargo executable found at `{}`".format(
1020 args = [self.cargo(), "build", "--manifest-path",
1021 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1022 for _ in range(0, self.verbose):
1023 args.append("--verbose")
1024 if self.use_locked_deps:
1025 args.append("--locked")
1026 if self.use_vendored_sources:
1027 args.append("--frozen")
1028 run(args, env=env, verbose=self.verbose)
1030 def build_triple(self):
1031 """Build triple as in LLVM
1033 Note that `default_build_triple` is moderately expensive,
1034 so use `self.build` where possible.
1036 config = self.get_toml('build')
1039 return default_build_triple(self.verbose)
1041 def check_submodule(self, module, slow_submodules):
1042 if not slow_submodules:
1043 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1044 cwd=os.path.join(self.rust_root, module),
1045 stdout=subprocess.PIPE)
1050 def update_submodule(self, module, checked_out, recorded_submodules):
1051 module_path = os.path.join(self.rust_root, module)
1053 if checked_out is not None:
1054 default_encoding = sys.getdefaultencoding()
1055 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1056 if recorded_submodules[module] == checked_out:
1059 print("Updating submodule", module)
1061 run(["git", "submodule", "-q", "sync", module],
1062 cwd=self.rust_root, verbose=self.verbose)
1064 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1065 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1066 update_args.append("--progress")
1067 update_args.append(module)
1069 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1070 except RuntimeError:
1071 print("Failed updating submodule. This is probably due to uncommitted local changes.")
1072 print('Either stash the changes by running "git stash" within the submodule\'s')
1073 print('directory, reset them by running "git reset --hard", or commit them.')
1074 print("To reset all submodules' changes run", end=" ")
1075 print('"git submodule foreach --recursive git reset --hard".')
1078 run(["git", "reset", "-q", "--hard"],
1079 cwd=module_path, verbose=self.verbose)
1080 run(["git", "clean", "-qdfx"],
1081 cwd=module_path, verbose=self.verbose)
1083 def update_submodules(self):
1084 """Update submodules"""
1085 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1086 self.get_toml('submodules') == "false":
1089 default_encoding = sys.getdefaultencoding()
1091 # check the existence and version of 'git' command
1092 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1093 self.git_version = distutils.version.LooseVersion(git_version_str)
1095 slow_submodules = self.get_toml('fast-submodules') == "false"
1098 print('Unconditionally updating submodules')
1100 print('Updating only changed submodules')
1101 default_encoding = sys.getdefaultencoding()
1102 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1103 # currently requires everything in a workspace to be "locally present" when starting a
1104 # build, and will give a hard error if any Cargo.toml files are missing.
1105 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1106 # share a workspace with any tools - maybe it could be excluded from the workspace?
1107 # That will still require cloning the submodules the second you check the standard
1108 # library, though...
1109 # FIXME: Is there a way to avoid hard-coding the submodules required?
1110 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1112 "src/tools/rust-installer",
1116 "library/backtrace",
1119 filtered_submodules = []
1120 submodules_names = []
1121 for module in submodules:
1122 check = self.check_submodule(module, slow_submodules)
1123 filtered_submodules.append((module, check))
1124 submodules_names.append(module)
1125 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1126 cwd=self.rust_root, stdout=subprocess.PIPE)
1127 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1128 # { filename: hash }
1129 recorded_submodules = {}
1130 for data in recorded:
1131 # [mode, kind, hash, filename]
1133 recorded_submodules[data[3]] = data[2]
1134 for module in filtered_submodules:
1135 self.update_submodule(module[0], module[1], recorded_submodules)
1136 print("Submodules updated in %.2f seconds" % (time() - start_time))
1138 def set_dist_environment(self, url):
1139 """Set download URL for normal environment"""
1140 if 'RUSTUP_DIST_SERVER' in os.environ:
1141 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1143 self._download_url = url
1145 def check_vendored_status(self):
1146 """Check that vendoring is configured properly"""
1147 vendor_dir = os.path.join(self.rust_root, 'vendor')
1148 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1149 if os.environ.get('USER') != os.environ['SUDO_USER']:
1150 self.use_vendored_sources = True
1151 print('info: looks like you are running this command under `sudo`')
1152 print(' and so in order to preserve your $HOME this will now')
1153 print(' use vendored sources by default.')
1154 if not os.path.exists(vendor_dir):
1155 print('error: vendoring required, but vendor directory does not exist.')
1156 print(' Run `cargo vendor` without sudo to initialize the '
1157 'vendor directory.')
1158 raise Exception("{} not found".format(vendor_dir))
1160 if self.use_vendored_sources:
1161 config = ("[source.crates-io]\n"
1162 "replace-with = 'vendored-sources'\n"
1163 "registry = 'https://example.com'\n"
1165 "[source.vendored-sources]\n"
1166 "directory = '{}/vendor'\n"
1167 .format(self.rust_root))
1168 if not os.path.exists('.cargo'):
1169 os.makedirs('.cargo')
1170 with output('.cargo/config') as cargo_config:
1171 cargo_config.write(config)
1173 print('info: using vendored source, but .cargo/config is already present.')
1174 print(' Reusing the current configuration file. But you may want to '
1175 'configure vendoring like this:')
1178 if os.path.exists('.cargo'):
1179 shutil.rmtree('.cargo')
1181 def ensure_vendored(self):
1182 """Ensure that the vendored sources are available if needed"""
1183 vendor_dir = os.path.join(self.rust_root, 'vendor')
1184 # Note that this does not handle updating the vendored dependencies if
1185 # the rust git repository is updated. Normal development usually does
1186 # not use vendoring, so hopefully this isn't too much of a problem.
1187 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1191 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1192 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1193 ], verbose=self.verbose, cwd=self.rust_root)
1196 def bootstrap(help_triggered):
1197 """Configure, fetch, build and run the initial bootstrap"""
1199 # If the user is asking for help, let them know that the whole download-and-build
1200 # process has to happen before anything is printed out.
1202 print("info: Downloading and building bootstrap before processing --help")
1203 print(" command. See src/bootstrap/README.md for help with common")
1206 parser = argparse.ArgumentParser(description='Build rust')
1207 parser.add_argument('--config')
1208 parser.add_argument('--build')
1209 parser.add_argument('--clean', action='store_true')
1210 parser.add_argument('-v', '--verbose', action='count', default=0)
1212 args = [a for a in sys.argv if a != '-h' and a != '--help']
1213 args, _ = parser.parse_known_args(args)
1215 # Configure initial bootstrap
1217 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1218 build.verbose = args.verbose
1219 build.clean = args.clean
1221 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1223 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1224 if not toml_path and os.path.exists('config.toml'):
1225 toml_path = 'config.toml'
1228 if not os.path.exists(toml_path):
1229 toml_path = os.path.join(build.rust_root, toml_path)
1231 with open(toml_path) as config:
1232 build.config_toml = config.read()
1234 profile = build.get_toml('profile')
1235 if profile is not None:
1236 include_file = 'config.{}.toml'.format(profile)
1237 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1238 include_path = os.path.join(include_dir, include_file)
1239 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1240 # specific key, so appending our defaults at the end allows the user to override them
1241 with open(include_path) as included_toml:
1242 build.config_toml += os.linesep + included_toml.read()
1244 config_verbose = build.get_toml('verbose', 'build')
1245 if config_verbose is not None:
1246 build.verbose = max(build.verbose, int(config_verbose))
1248 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1250 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1252 build.check_vendored_status()
1254 build_dir = build.get_toml('build-dir', 'build') or 'build'
1255 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1257 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1259 build.checksums_sha256 = data["checksums_sha256"]
1260 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1261 if data.get("rustfmt") is not None:
1262 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1264 build.set_dist_environment(data["dist_server"])
1266 build.build = args.build or build.build_triple()
1268 # Acquire the lock before doing any build actions
1269 # The lock is released when `lock` is dropped
1270 if not os.path.exists(build.build_dir):
1271 os.makedirs(build.build_dir)
1272 lock = acquire_lock(build.build_dir)
1273 build.update_submodules()
1275 # Fetch/build the bootstrap
1276 build.download_toolchain()
1277 # Download the master compiler if `download-rustc` is set
1278 build.maybe_download_ci_toolchain()
1280 build.ensure_vendored()
1281 build.build_bootstrap()
1285 args = [build.bootstrap_binary()]
1286 args.extend(sys.argv[1:])
1287 env = os.environ.copy()
1288 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1289 env["BOOTSTRAP_PYTHON"] = sys.executable
1290 env["BUILD_DIR"] = build.build_dir
1291 env["RUSTC_BOOTSTRAP"] = '1'
1293 env["BOOTSTRAP_CONFIG"] = toml_path
1294 if build.rustc_commit is not None:
1295 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1296 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1300 """Entry point for the bootstrap process"""
1303 # x.py help <cmd> ...
1304 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1305 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1308 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1310 bootstrap(help_triggered)
1311 if not help_triggered:
1312 print("Build completed successfully in {}".format(
1313 format_build_time(time() - start_time)))
1314 except (SystemExit, KeyboardInterrupt) as error:
1315 if hasattr(error, 'code') and isinstance(error.code, int):
1316 exit_code = error.code
1320 if not help_triggered:
1321 print("Build completed unsuccessfully in {}".format(
1322 format_build_time(time() - start_time)))
1326 if __name__ == '__main__':