1 from __future__ import absolute_import, division, print_function
5 import distutils.version
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
25 path = os.path.join(build_dir, "lock.db")
27 con = sqlite3.Connection(path, timeout=0)
29 curs.execute("BEGIN EXCLUSIVE")
30 # The lock is released when the cursor is dropped
32 # If the database is busy then lock has already been acquired
33 # so we wait for the lock.
34 # We retry every quarter second so that execution is passed back to python
35 # so that it can handle signals
36 except sqlite3.OperationalError:
39 print("Waiting for lock on build directory")
40 con = sqlite3.Connection(path, timeout=0.25)
44 curs.execute("BEGIN EXCLUSIVE")
45 except sqlite3.OperationalError:
49 print("warning: sqlite3 not available in python, skipping build directory lock")
50 print("please file an issue on rust-lang/rust")
51 print("this is not a problem for non-concurrent x.py invocations")
56 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
57 temp_path = temp_file.name
58 with tarfile.open(temp_path, "w:xz"):
61 except tarfile.CompressionError:
64 def get(base, url, path, checksums, verbose=False, do_verify=True):
65 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
66 temp_path = temp_file.name
70 if url not in checksums:
71 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
72 sha256 = checksums[url]
73 if os.path.exists(path):
74 if verify(path, sha256, False):
76 print("using already-download file", path)
80 print("ignoring already-download file",
81 path, "due to failed verification")
83 download(temp_path, "{}/{}".format(base, url), True, verbose)
84 if do_verify and not verify(temp_path, sha256, verbose):
85 raise RuntimeError("failed verification")
87 print("moving {} to {}".format(temp_path, path))
88 shutil.move(temp_path, path)
90 if os.path.isfile(temp_path):
92 print("removing", temp_path)
96 def download(path, url, probably_big, verbose):
99 _download(path, url, probably_big, verbose, True)
102 print("\nspurious failure, trying again")
103 _download(path, url, probably_big, verbose, False)
106 def _download(path, url, probably_big, verbose, exception):
107 if probably_big or verbose:
108 print("downloading {}".format(url))
109 # see https://serverfault.com/questions/301128/how-to-download
110 if sys.platform == 'win32':
111 run(["PowerShell.exe", "/nologo", "-Command",
112 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
113 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
117 if probably_big or verbose:
121 require(["curl", "--version"])
123 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
124 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
125 "--retry", "3", "-Sf", "-o", path, url],
130 def verify(path, expected, verbose):
131 """Check if the sha256 sum of the given path is valid"""
133 print("verifying", path)
134 with open(path, "rb") as source:
135 found = hashlib.sha256(source.read()).hexdigest()
136 verified = found == expected
138 print("invalid checksum:\n"
140 " expected: {}".format(found, expected))
144 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
145 """Unpack the given tarball file"""
146 print("extracting", tarball)
147 fname = os.path.basename(tarball).replace(tarball_suffix, "")
148 with contextlib.closing(tarfile.open(tarball)) as tar:
149 for member in tar.getnames():
150 if "/" not in member:
152 name = member.replace(fname + "/", "", 1)
153 if match is not None and not name.startswith(match):
155 name = name[len(match) + 1:]
157 dst_path = os.path.join(dst, name)
159 print(" extracting", member)
160 tar.extract(member, dst)
161 src_path = os.path.join(dst, member)
162 if os.path.isdir(src_path) and os.path.exists(dst_path):
164 shutil.move(src_path, dst_path)
165 shutil.rmtree(os.path.join(dst, fname))
168 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
169 """Run a child program in a new process"""
171 print("running: " + ' '.join(args))
173 # Use Popen here instead of call() as it apparently allows powershell on
174 # Windows to not lock up waiting for input presumably.
175 ret = subprocess.Popen(args, **kwargs)
178 err = "failed to run: " + ' '.join(args)
179 if verbose or exception:
180 raise RuntimeError(err)
181 # For most failures, we definitely do want to print this error, or the user will have no
182 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
183 # have already printed an error above, so there's no need to print the exact command we're
191 def require(cmd, exit=True):
192 '''Run a command, returning its output.
194 If `exit` is `True`, exit the process.
195 Otherwise, return None.'''
197 return subprocess.check_output(cmd).strip()
198 except (subprocess.CalledProcessError, OSError) as exc:
201 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
202 print("Please make sure it's installed and in the path.")
206 def format_build_time(duration):
207 """Return a nicer format for build time
209 >>> format_build_time('300')
212 return str(datetime.timedelta(seconds=int(duration)))
215 def default_build_triple(verbose):
216 """Build triple as in LLVM"""
217 # If the user already has a host build triple with an existing `rustc`
218 # install, use their preference. This fixes most issues with Windows builds
219 # being detected as GNU instead of MSVC.
220 default_encoding = sys.getdefaultencoding()
222 version = subprocess.check_output(["rustc", "--version", "--verbose"],
223 stderr=subprocess.DEVNULL)
224 version = version.decode(default_encoding)
225 host = next(x for x in version.split('\n') if x.startswith("host: "))
226 triple = host.split("host: ")[1]
228 print("detected default triple {} from pre-installed rustc".format(triple))
230 except Exception as e:
232 print("pre-installed rustc not detected: {}".format(e))
233 print("falling back to auto-detect")
235 required = sys.platform != 'win32'
236 ostype = require(["uname", "-s"], exit=required)
237 cputype = require(['uname', '-m'], exit=required)
239 # If we do not have `uname`, assume Windows.
240 if ostype is None or cputype is None:
241 return 'x86_64-pc-windows-msvc'
243 ostype = ostype.decode(default_encoding)
244 cputype = cputype.decode(default_encoding)
246 # The goal here is to come up with the same triple as LLVM would,
247 # at least for the subset of platforms we're willing to target.
249 'Darwin': 'apple-darwin',
250 'DragonFly': 'unknown-dragonfly',
251 'FreeBSD': 'unknown-freebsd',
252 'Haiku': 'unknown-haiku',
253 'NetBSD': 'unknown-netbsd',
254 'OpenBSD': 'unknown-openbsd'
257 # Consider the direct transformation first and then the special cases
258 if ostype in ostype_mapper:
259 ostype = ostype_mapper[ostype]
260 elif ostype == 'Linux':
261 os_from_sp = subprocess.check_output(
262 ['uname', '-o']).strip().decode(default_encoding)
263 if os_from_sp == 'Android':
264 ostype = 'linux-android'
266 ostype = 'unknown-linux-gnu'
267 elif ostype == 'SunOS':
268 ostype = 'pc-solaris'
269 # On Solaris, uname -m will return a machine classification instead
270 # of a cpu type, so uname -p is recommended instead. However, the
271 # output from that option is too generic for our purposes (it will
272 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
273 # must be used instead.
274 cputype = require(['isainfo', '-k']).decode(default_encoding)
275 # sparc cpus have sun as a target vendor
276 if 'sparc' in cputype:
277 ostype = 'sun-solaris'
278 elif ostype.startswith('MINGW'):
279 # msys' `uname` does not print gcc configuration, but prints msys
280 # configuration. so we cannot believe `uname -m`:
281 # msys1 is always i686 and msys2 is always x86_64.
282 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
284 ostype = 'pc-windows-gnu'
286 if os.environ.get('MSYSTEM') == 'MINGW64':
288 elif ostype.startswith('MSYS'):
289 ostype = 'pc-windows-gnu'
290 elif ostype.startswith('CYGWIN_NT'):
292 if ostype.endswith('WOW64'):
294 ostype = 'pc-windows-gnu'
295 elif sys.platform == 'win32':
296 # Some Windows platforms might have a `uname` command that returns a
297 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
298 # these cases, fall back to using sys.platform.
299 return 'x86_64-pc-windows-msvc'
301 err = "unknown OS type: {}".format(ostype)
304 if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
305 cputype = subprocess.check_output(
306 ['uname', '-p']).strip().decode(default_encoding)
309 'aarch64': 'aarch64',
317 'powerpc': 'powerpc',
318 'powerpc64': 'powerpc64',
319 'powerpc64le': 'powerpc64le',
321 'ppc64': 'powerpc64',
322 'ppc64le': 'powerpc64le',
323 'riscv64': 'riscv64gc',
331 # Consider the direct transformation first and then the special cases
332 if cputype in cputype_mapper:
333 cputype = cputype_mapper[cputype]
334 elif cputype in {'xscale', 'arm'}:
336 if ostype == 'linux-android':
337 ostype = 'linux-androideabi'
338 elif ostype == 'unknown-freebsd':
339 cputype = subprocess.check_output(
340 ['uname', '-p']).strip().decode(default_encoding)
341 ostype = 'unknown-freebsd'
342 elif cputype == 'armv6l':
344 if ostype == 'linux-android':
345 ostype = 'linux-androideabi'
348 elif cputype in {'armv7l', 'armv8l'}:
350 if ostype == 'linux-android':
351 ostype = 'linux-androideabi'
354 elif cputype == 'mips':
355 if sys.byteorder == 'big':
357 elif sys.byteorder == 'little':
360 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
361 elif cputype == 'mips64':
362 if sys.byteorder == 'big':
364 elif sys.byteorder == 'little':
367 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
368 # only the n64 ABI is supported, indicate it
370 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
373 err = "unknown cpu type: {}".format(cputype)
376 return "{}-{}".format(cputype, ostype)
379 @contextlib.contextmanager
380 def output(filepath):
381 tmp = filepath + '.tmp'
382 with open(tmp, 'w') as f:
385 if os.path.exists(filepath):
386 os.remove(filepath) # PermissionError/OSError on Win32 if in use
388 shutil.copy2(tmp, filepath)
391 os.rename(tmp, filepath)
394 class Stage0Toolchain:
395 def __init__(self, stage0_payload):
396 self.date = stage0_payload["date"]
397 self.version = stage0_payload["version"]
400 return self.version + "-" + self.date
403 class RustBuild(object):
404 """Provide all the methods required to build Rust"""
406 self.checksums_sha256 = {}
407 self.stage0_compiler = None
408 self.stage0_rustfmt = None
409 self._download_url = ''
413 self.config_toml = ''
415 self.use_locked_deps = ''
416 self.use_vendored_sources = ''
418 self.git_version = None
419 self.nix_deps_dir = None
420 self.rustc_commit = None
422 def download_toolchain(self, stage0=True, rustc_channel=None):
423 """Fetch the build system for Rust, written in Rust
425 This method will build a cache directory, then it will fetch the
426 tarball which has the stage0 compiler used to then bootstrap the Rust
429 Each downloaded tarball is extracted, after that, the script
430 will move all the content to the right place.
432 if rustc_channel is None:
433 rustc_channel = self.stage0_compiler.version
434 bin_root = self.bin_root(stage0)
436 key = self.stage0_compiler.date
438 key += str(self.rustc_commit)
439 if self.rustc(stage0).startswith(bin_root) and \
440 (not os.path.exists(self.rustc(stage0)) or
441 self.program_out_of_date(self.rustc_stamp(stage0), key)):
442 if os.path.exists(bin_root):
443 shutil.rmtree(bin_root)
444 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
445 filename = "rust-std-{}-{}{}".format(
446 rustc_channel, self.build, tarball_suffix)
447 pattern = "rust-std-{}".format(self.build)
448 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
449 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
451 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
452 # download-rustc doesn't need its own cargo, it can just use beta's.
454 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
456 self._download_component_helper(filename, "cargo", tarball_suffix)
457 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
459 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
460 self._download_component_helper(
461 filename, "rustc-dev", tarball_suffix, stage0
464 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
465 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
466 lib_dir = "{}/lib".format(bin_root)
467 for lib in os.listdir(lib_dir):
468 if lib.endswith(".so"):
469 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
470 with output(self.rustc_stamp(stage0)) as rust_stamp:
471 rust_stamp.write(key)
473 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
474 not os.path.exists(self.rustfmt())
475 or self.program_out_of_date(
476 self.rustfmt_stamp(),
477 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
480 if self.stage0_rustfmt is not None:
481 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
482 filename = "rustfmt-{}-{}{}".format(
483 self.stage0_rustfmt.version, self.build, tarball_suffix,
485 self._download_component_helper(
486 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
488 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
489 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
490 with output(self.rustfmt_stamp()) as rustfmt_stamp:
491 rustfmt_stamp.write(self.stage0_rustfmt.channel())
493 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
494 if self.downloading_llvm() and stage0:
495 # We want the most recent LLVM submodule update to avoid downloading
496 # LLVM more often than necessary.
498 # This git command finds that commit SHA, looking for bors-authored
499 # commits that modified src/llvm-project or other relevant version
502 # This works even in a repository that has not yet initialized
504 top_level = subprocess.check_output([
505 "git", "rev-parse", "--show-toplevel",
506 ]).decode(sys.getdefaultencoding()).strip()
507 llvm_sha = subprocess.check_output([
508 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
509 "--first-parent", "HEAD",
511 "{}/src/llvm-project".format(top_level),
512 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
513 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
514 "{}/src/version".format(top_level)
515 ]).decode(sys.getdefaultencoding()).strip()
516 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
517 llvm_root = self.llvm_root()
518 llvm_lib = os.path.join(llvm_root, "lib")
519 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
520 self._download_ci_llvm(llvm_sha, llvm_assertions)
521 for binary in ["llvm-config", "FileCheck"]:
522 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
523 for lib in os.listdir(llvm_lib):
524 if lib.endswith(".so"):
525 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
526 with output(self.llvm_stamp()) as llvm_stamp:
527 llvm_stamp.write(llvm_sha + str(llvm_assertions))
529 def downloading_llvm(self):
530 opt = self.get_toml('download-ci-llvm', 'llvm')
531 # This is currently all tier 1 targets and tier 2 targets with host tools
532 # (since others may not have CI artifacts)
533 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
534 supported_platforms = [
536 "aarch64-unknown-linux-gnu",
537 "i686-pc-windows-gnu",
538 "i686-pc-windows-msvc",
539 "i686-unknown-linux-gnu",
540 "x86_64-unknown-linux-gnu",
541 "x86_64-apple-darwin",
542 "x86_64-pc-windows-gnu",
543 "x86_64-pc-windows-msvc",
544 # tier 2 with host tools
545 "aarch64-apple-darwin",
546 "aarch64-pc-windows-msvc",
547 "aarch64-unknown-linux-musl",
548 "arm-unknown-linux-gnueabi",
549 "arm-unknown-linux-gnueabihf",
550 "armv7-unknown-linux-gnueabihf",
551 "mips-unknown-linux-gnu",
552 "mips64-unknown-linux-gnuabi64",
553 "mips64el-unknown-linux-gnuabi64",
554 "mipsel-unknown-linux-gnu",
555 "powerpc-unknown-linux-gnu",
556 "powerpc64-unknown-linux-gnu",
557 "powerpc64le-unknown-linux-gnu",
558 "riscv64gc-unknown-linux-gnu",
559 "s390x-unknown-linux-gnu",
560 "x86_64-unknown-freebsd",
561 "x86_64-unknown-illumos",
562 "x86_64-unknown-linux-musl",
563 "x86_64-unknown-netbsd",
565 return opt == "true" \
566 or (opt == "if-available" and self.build in supported_platforms)
568 def _download_component_helper(
569 self, filename, pattern, tarball_suffix, stage0=True, key=None
573 key = self.stage0_compiler.date
575 key = self.rustc_commit
576 cache_dst = os.path.join(self.build_dir, "cache")
577 rustc_cache = os.path.join(cache_dst, key)
578 if not os.path.exists(rustc_cache):
579 os.makedirs(rustc_cache)
582 base = self._download_url
583 url = "dist/{}".format(key)
585 base = "https://ci-artifacts.rust-lang.org"
586 url = "rustc-builds/{}".format(self.rustc_commit)
587 tarball = os.path.join(rustc_cache, filename)
588 if not os.path.exists(tarball):
591 "{}/{}".format(url, filename),
593 self.checksums_sha256,
594 verbose=self.verbose,
597 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
599 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
601 print("error: could not find commit hash for downloading LLVM")
602 print("help: maybe your repository history is too shallow?")
603 print("help: consider disabling `download-ci-llvm`")
604 print("help: or fetch enough history to include one upstream commit")
606 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
607 cache_dst = os.path.join(self.build_dir, "cache")
608 rustc_cache = os.path.join(cache_dst, cache_prefix)
609 if not os.path.exists(rustc_cache):
610 os.makedirs(rustc_cache)
612 base = "https://ci-artifacts.rust-lang.org"
613 url = "rustc-builds/{}".format(llvm_sha)
615 url = url.replace('rustc-builds', 'rustc-builds-alt')
616 # ci-artifacts are only stored as .xz, not .gz
618 print("error: XZ support is required to download LLVM")
619 print("help: consider disabling `download-ci-llvm` or using python3")
621 tarball_suffix = '.tar.xz'
622 filename = "rust-dev-nightly-" + self.build + tarball_suffix
623 tarball = os.path.join(rustc_cache, filename)
624 if not os.path.exists(tarball):
627 "{}/{}".format(url, filename),
629 self.checksums_sha256,
630 verbose=self.verbose,
633 unpack(tarball, tarball_suffix, self.llvm_root(),
635 verbose=self.verbose)
637 def fix_bin_or_dylib(self, fname):
638 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
639 or the RPATH section, to fix the dynamic library search path
641 This method is only required on NixOS and uses the PatchELF utility to
642 change the interpreter/RPATH of ELF executables.
644 Please see https://nixos.org/patchelf.html for more information
646 default_encoding = sys.getdefaultencoding()
648 ostype = subprocess.check_output(
649 ['uname', '-s']).strip().decode(default_encoding)
650 except subprocess.CalledProcessError:
652 except OSError as reason:
653 if getattr(reason, 'winerror', None) is not None:
657 if ostype != "Linux":
660 # If the user has asked binaries to be patched for Nix, then
661 # don't check for NixOS or `/lib`, just continue to the patching.
662 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
663 # Use `/etc/os-release` instead of `/etc/NIXOS`.
664 # The latter one does not exist on NixOS when using tmpfs as root.
666 with open("/etc/os-release", "r") as f:
667 if not any(line.strip() == "ID=nixos" for line in f):
669 except FileNotFoundError:
671 if os.path.exists("/lib"):
674 # At this point we're pretty sure the user is running NixOS or
676 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
677 print(nix_os_msg, fname)
679 # Only build `.nix-deps` once.
680 nix_deps_dir = self.nix_deps_dir
682 # Run `nix-build` to "build" each dependency (which will likely reuse
683 # the existing `/nix/store` copy, or at most download a pre-built copy).
685 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
686 # directory, but still reference the actual `/nix/store` path in the rpath
687 # as it makes it significantly more robust against changes to the location of
688 # the `.nix-deps` location.
690 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
691 # zlib: Needed as a system dependency of `libLLVM-*.so`.
692 # patchelf: Needed for patching ELF binaries (see doc comment above).
693 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
695 with (import <nixpkgs> {});
697 name = "rust-stage0-dependencies";
706 subprocess.check_output([
707 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
709 except subprocess.CalledProcessError as reason:
710 print("warning: failed to call nix-build:", reason)
712 self.nix_deps_dir = nix_deps_dir
714 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
716 # Relative default, all binary and dynamic libraries we ship
717 # appear to have this (even when `../lib` is redundant).
719 os.path.join(os.path.realpath(nix_deps_dir), "lib")
721 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
722 if not fname.endswith(".so"):
723 # Finally, set the corret .interp for binaries
724 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
725 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
728 subprocess.check_output([patchelf] + patchelf_args + [fname])
729 except subprocess.CalledProcessError as reason:
730 print("warning: failed to call patchelf:", reason)
733 # If `download-rustc` is set, download the most recent commit with CI artifacts
734 def maybe_download_ci_toolchain(self):
735 # If `download-rustc` is not set, default to rebuilding.
736 download_rustc = self.get_toml("download-rustc", section="rust")
737 if download_rustc is None or download_rustc == "false":
739 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
741 # Handle running from a directory other than the top level
742 rev_parse = ["git", "rev-parse", "--show-toplevel"]
743 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
744 compiler = "{}/compiler/".format(top_level)
745 library = "{}/library/".format(top_level)
747 # Look for a version to compare to based on the current commit.
748 # Only commits merged by bors will have CI artifacts.
750 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
751 "--first-parent", "HEAD"
753 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
755 print("error: could not find commit hash for downloading rustc")
756 print("help: maybe your repository history is too shallow?")
757 print("help: consider disabling `download-rustc`")
758 print("help: or fetch enough history to include one upstream commit")
761 # Warn if there were changes to the compiler or standard library since the ancestor commit.
762 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
764 if download_rustc == "if-unchanged":
766 print("warning: saw changes to compiler/ or library/ since {}; " \
767 "ignoring `download-rustc`".format(commit))
769 print("warning: `download-rustc` is enabled, but there are changes to " \
770 "compiler/ or library/")
773 print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
774 self.rustc_commit = commit
775 # FIXME: support downloading artifacts from the beta channel
776 self.download_toolchain(False, "nightly")
778 def rustc_stamp(self, stage0):
779 """Return the path for .rustc-stamp at the given stage
782 >>> rb.build_dir = "build"
783 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
785 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
788 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
790 def rustfmt_stamp(self):
791 """Return the path for .rustfmt-stamp
794 >>> rb.build_dir = "build"
795 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
798 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
800 def llvm_stamp(self):
801 """Return the path for .rustfmt-stamp
804 >>> rb.build_dir = "build"
805 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
808 return os.path.join(self.llvm_root(), '.llvm-stamp')
811 def program_out_of_date(self, stamp_path, key):
812 """Check if the given program stamp is out of date"""
813 if not os.path.exists(stamp_path) or self.clean:
815 with open(stamp_path, 'r') as stamp:
816 return key != stamp.read()
818 def bin_root(self, stage0):
819 """Return the binary root directory for the given stage
822 >>> rb.build_dir = "build"
823 >>> rb.bin_root(True) == os.path.join("build", "stage0")
825 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
828 When the 'build' property is given should be a nested directory:
830 >>> rb.build = "devel"
831 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
838 return os.path.join(self.build_dir, self.build, subdir)
841 """Return the CI LLVM root directory
844 >>> rb.build_dir = "build"
845 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
848 When the 'build' property is given should be a nested directory:
850 >>> rb.build = "devel"
851 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
854 return os.path.join(self.build_dir, self.build, "ci-llvm")
856 def get_toml(self, key, section=None):
857 """Returns the value of the given key in config.toml, otherwise returns None
860 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
861 >>> rb.get_toml("key2")
864 If the key does not exists, the result is None:
866 >>> rb.get_toml("key3") is None
869 Optionally also matches the section the key appears in
871 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
872 >>> rb.get_toml('key', 'a')
874 >>> rb.get_toml('key', 'b')
876 >>> rb.get_toml('key', 'c') is None
879 >>> rb.config_toml = 'key1 = true'
880 >>> rb.get_toml("key1")
885 for line in self.config_toml.splitlines():
886 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
887 if section_match is not None:
888 cur_section = section_match.group(1)
890 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
891 if match is not None:
892 value = match.group(1)
893 if section is None or section == cur_section:
894 return self.get_string(value) or value.strip()
898 """Return config path for cargo"""
899 return self.program_config('cargo')
901 def rustc(self, stage0):
902 """Return config path for rustc"""
903 return self.program_config('rustc', stage0)
906 """Return config path for rustfmt"""
907 if self.stage0_rustfmt is None:
909 return self.program_config('rustfmt')
911 def program_config(self, program, stage0=True):
912 """Return config path for the given program at the given stage
915 >>> rb.config_toml = 'rustc = "rustc"\\n'
916 >>> rb.program_config('rustc')
918 >>> rb.config_toml = ''
919 >>> cargo_path = rb.program_config('cargo', True)
920 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
923 >>> cargo_path = rb.program_config('cargo', False)
924 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
928 config = self.get_toml(program)
930 return os.path.expanduser(config)
931 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
932 program, self.exe_suffix()))
935 def get_string(line):
936 """Return the value between double quotes
938 >>> RustBuild.get_string(' "devel" ')
940 >>> RustBuild.get_string(" 'devel' ")
942 >>> RustBuild.get_string('devel') is None
944 >>> RustBuild.get_string(' "devel ')
947 start = line.find('"')
949 end = start + 1 + line[start + 1:].find('"')
950 return line[start + 1:end]
951 start = line.find('\'')
953 end = start + 1 + line[start + 1:].find('\'')
954 return line[start + 1:end]
959 """Return a suffix for executables"""
960 if sys.platform == 'win32':
964 def bootstrap_binary(self):
965 """Return the path of the bootstrap binary
968 >>> rb.build_dir = "build"
969 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
970 ... "debug", "bootstrap")
973 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
975 def build_bootstrap(self):
976 """Build bootstrap"""
977 print("Building rustbuild")
978 build_dir = os.path.join(self.build_dir, "bootstrap")
979 if self.clean and os.path.exists(build_dir):
980 shutil.rmtree(build_dir)
981 env = os.environ.copy()
982 # `CARGO_BUILD_TARGET` breaks bootstrap build.
983 # See also: <https://github.com/rust-lang/rust/issues/70208>.
984 if "CARGO_BUILD_TARGET" in env:
985 del env["CARGO_BUILD_TARGET"]
986 env["CARGO_TARGET_DIR"] = build_dir
987 env["RUSTC"] = self.rustc(True)
988 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
989 (os.pathsep + env["LD_LIBRARY_PATH"]) \
990 if "LD_LIBRARY_PATH" in env else ""
991 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
992 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
993 if "DYLD_LIBRARY_PATH" in env else ""
994 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
995 (os.pathsep + env["LIBRARY_PATH"]) \
996 if "LIBRARY_PATH" in env else ""
998 # preserve existing RUSTFLAGS
999 env.setdefault("RUSTFLAGS", "")
1000 build_section = "target.{}".format(self.build)
1001 target_features = []
1002 if self.get_toml("crt-static", build_section) == "true":
1003 target_features += ["+crt-static"]
1004 elif self.get_toml("crt-static", build_section) == "false":
1005 target_features += ["-crt-static"]
1007 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1008 target_linker = self.get_toml("linker", build_section)
1009 if target_linker is not None:
1010 env["RUSTFLAGS"] += " -C linker=" + target_linker
1011 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1012 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1013 if self.get_toml("deny-warnings", "rust") != "false":
1014 env["RUSTFLAGS"] += " -Dwarnings"
1016 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1017 os.pathsep + env["PATH"]
1018 if not os.path.isfile(self.cargo()):
1019 raise Exception("no cargo executable found at `{}`".format(
1021 args = [self.cargo(), "build", "--manifest-path",
1022 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1023 for _ in range(0, self.verbose):
1024 args.append("--verbose")
1025 if self.use_locked_deps:
1026 args.append("--locked")
1027 if self.use_vendored_sources:
1028 args.append("--frozen")
1029 run(args, env=env, verbose=self.verbose)
1031 def build_triple(self):
1032 """Build triple as in LLVM
1034 Note that `default_build_triple` is moderately expensive,
1035 so use `self.build` where possible.
1037 config = self.get_toml('build')
1040 return default_build_triple(self.verbose)
1042 def check_submodule(self, module, slow_submodules):
1043 if not slow_submodules:
1044 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1045 cwd=os.path.join(self.rust_root, module),
1046 stdout=subprocess.PIPE)
1051 def update_submodule(self, module, checked_out, recorded_submodules):
1052 module_path = os.path.join(self.rust_root, module)
1054 if checked_out is not None:
1055 default_encoding = sys.getdefaultencoding()
1056 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1057 if recorded_submodules[module] == checked_out:
1060 print("Updating submodule", module)
1062 run(["git", "submodule", "-q", "sync", module],
1063 cwd=self.rust_root, verbose=self.verbose)
1065 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1066 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1067 update_args.append("--progress")
1068 update_args.append(module)
1070 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1071 except RuntimeError:
1072 print("Failed updating submodule. This is probably due to uncommitted local changes.")
1073 print('Either stash the changes by running "git stash" within the submodule\'s')
1074 print('directory, reset them by running "git reset --hard", or commit them.')
1075 print("To reset all submodules' changes run", end=" ")
1076 print('"git submodule foreach --recursive git reset --hard".')
1079 run(["git", "reset", "-q", "--hard"],
1080 cwd=module_path, verbose=self.verbose)
1081 run(["git", "clean", "-qdfx"],
1082 cwd=module_path, verbose=self.verbose)
1084 def update_submodules(self):
1085 """Update submodules"""
1086 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1087 self.get_toml('submodules') == "false":
1090 default_encoding = sys.getdefaultencoding()
1092 # check the existence and version of 'git' command
1093 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1094 self.git_version = distutils.version.LooseVersion(git_version_str)
1096 slow_submodules = self.get_toml('fast-submodules') == "false"
1099 print('Unconditionally updating submodules')
1101 print('Updating only changed submodules')
1102 default_encoding = sys.getdefaultencoding()
1103 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1104 # currently requires everything in a workspace to be "locally present" when starting a
1105 # build, and will give a hard error if any Cargo.toml files are missing.
1106 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1107 # share a workspace with any tools - maybe it could be excluded from the workspace?
1108 # That will still require cloning the submodules the second you check the standard
1109 # library, though...
1110 # FIXME: Is there a way to avoid hard-coding the submodules required?
1111 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1113 "src/tools/rust-installer",
1117 "library/backtrace",
1120 filtered_submodules = []
1121 submodules_names = []
1122 for module in submodules:
1123 check = self.check_submodule(module, slow_submodules)
1124 filtered_submodules.append((module, check))
1125 submodules_names.append(module)
1126 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1127 cwd=self.rust_root, stdout=subprocess.PIPE)
1128 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1129 # { filename: hash }
1130 recorded_submodules = {}
1131 for data in recorded:
1132 # [mode, kind, hash, filename]
1134 recorded_submodules[data[3]] = data[2]
1135 for module in filtered_submodules:
1136 self.update_submodule(module[0], module[1], recorded_submodules)
1137 print(" Submodules updated in %.2f seconds" % (time() - start_time))
1139 def set_dist_environment(self, url):
1140 """Set download URL for normal environment"""
1141 if 'RUSTUP_DIST_SERVER' in os.environ:
1142 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1144 self._download_url = url
1146 def check_vendored_status(self):
1147 """Check that vendoring is configured properly"""
1148 vendor_dir = os.path.join(self.rust_root, 'vendor')
1149 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1150 if os.environ.get('USER') != os.environ['SUDO_USER']:
1151 self.use_vendored_sources = True
1152 print('info: looks like you are running this command under `sudo`')
1153 print(' and so in order to preserve your $HOME this will now')
1154 print(' use vendored sources by default.')
1155 if not os.path.exists(vendor_dir):
1156 print('error: vendoring required, but vendor directory does not exist.')
1157 print(' Run `cargo vendor` without sudo to initialize the '
1158 'vendor directory.')
1159 raise Exception("{} not found".format(vendor_dir))
1161 if self.use_vendored_sources:
1162 config = ("[source.crates-io]\n"
1163 "replace-with = 'vendored-sources'\n"
1164 "registry = 'https://example.com'\n"
1166 "[source.vendored-sources]\n"
1167 "directory = '{}/vendor'\n"
1168 .format(self.rust_root))
1169 if not os.path.exists('.cargo'):
1170 os.makedirs('.cargo')
1171 with output('.cargo/config') as cargo_config:
1172 cargo_config.write(config)
1174 print('info: using vendored source, but .cargo/config is already present.')
1175 print(' Reusing the current configuration file. But you may want to '
1176 'configure vendoring like this:')
1179 if os.path.exists('.cargo'):
1180 shutil.rmtree('.cargo')
1182 def ensure_vendored(self):
1183 """Ensure that the vendored sources are available if needed"""
1184 vendor_dir = os.path.join(self.rust_root, 'vendor')
1185 # Note that this does not handle updating the vendored dependencies if
1186 # the rust git repository is updated. Normal development usually does
1187 # not use vendoring, so hopefully this isn't too much of a problem.
1188 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1192 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1193 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1194 ], verbose=self.verbose, cwd=self.rust_root)
1197 def bootstrap(help_triggered):
1198 """Configure, fetch, build and run the initial bootstrap"""
1200 # If the user is asking for help, let them know that the whole download-and-build
1201 # process has to happen before anything is printed out.
1203 print("info: Downloading and building bootstrap before processing --help")
1204 print(" command. See src/bootstrap/README.md for help with common")
1207 parser = argparse.ArgumentParser(description='Build rust')
1208 parser.add_argument('--config')
1209 parser.add_argument('--build')
1210 parser.add_argument('--clean', action='store_true')
1211 parser.add_argument('-v', '--verbose', action='count', default=0)
1213 args = [a for a in sys.argv if a != '-h' and a != '--help']
1214 args, _ = parser.parse_known_args(args)
1216 # Configure initial bootstrap
1218 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1219 build.verbose = args.verbose
1220 build.clean = args.clean
1222 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1224 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1225 if not toml_path and os.path.exists('config.toml'):
1226 toml_path = 'config.toml'
1229 if not os.path.exists(toml_path):
1230 toml_path = os.path.join(build.rust_root, toml_path)
1232 with open(toml_path) as config:
1233 build.config_toml = config.read()
1235 profile = build.get_toml('profile')
1236 if profile is not None:
1237 include_file = 'config.{}.toml'.format(profile)
1238 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1239 include_path = os.path.join(include_dir, include_file)
1240 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1241 # specific key, so appending our defaults at the end allows the user to override them
1242 with open(include_path) as included_toml:
1243 build.config_toml += os.linesep + included_toml.read()
1245 config_verbose = build.get_toml('verbose', 'build')
1246 if config_verbose is not None:
1247 build.verbose = max(build.verbose, int(config_verbose))
1249 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1251 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1253 build.check_vendored_status()
1255 build_dir = build.get_toml('build-dir', 'build') or 'build'
1256 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1258 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1260 build.checksums_sha256 = data["checksums_sha256"]
1261 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1262 if data.get("rustfmt") is not None:
1263 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1265 build.set_dist_environment(data["dist_server"])
1267 build.build = args.build or build.build_triple()
1269 # Acquire the lock before doing any build actions
1270 # The lock is released when `lock` is dropped
1271 if not os.path.exists(build.build_dir):
1272 os.makedirs(build.build_dir)
1273 lock = acquire_lock(build.build_dir)
1274 build.update_submodules()
1276 # Fetch/build the bootstrap
1277 build.download_toolchain()
1278 # Download the master compiler if `download-rustc` is set
1279 build.maybe_download_ci_toolchain()
1281 build.ensure_vendored()
1282 build.build_bootstrap()
1286 args = [build.bootstrap_binary()]
1287 args.extend(sys.argv[1:])
1288 env = os.environ.copy()
1289 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1290 env["BOOTSTRAP_PYTHON"] = sys.executable
1291 env["BUILD_DIR"] = build.build_dir
1292 env["RUSTC_BOOTSTRAP"] = '1'
1294 env["BOOTSTRAP_CONFIG"] = toml_path
1295 if build.rustc_commit is not None:
1296 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1297 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1301 """Entry point for the bootstrap process"""
1304 # x.py help <cmd> ...
1305 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1306 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1309 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1311 bootstrap(help_triggered)
1312 if not help_triggered:
1313 print("Build completed successfully in {}".format(
1314 format_build_time(time() - start_time)))
1315 except (SystemExit, KeyboardInterrupt) as error:
1316 if hasattr(error, 'code') and isinstance(error.code, int):
1317 exit_code = error.code
1321 if not help_triggered:
1322 print("Build completed unsuccessfully in {}".format(
1323 format_build_time(time() - start_time)))
1327 if __name__ == '__main__':