1 from __future__ import absolute_import, division, print_function
5 import distutils.version
16 from time import time, sleep
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
25 path = os.path.join(build_dir, "lock.db")
27 con = sqlite3.Connection(path, timeout=0)
29 curs.execute("BEGIN EXCLUSIVE")
30 # The lock is released when the cursor is dropped
32 # If the database is busy then lock has already been acquired
33 # so we wait for the lock.
34 # We retry every quarter second so that execution is passed back to python
35 # so that it can handle signals
36 except sqlite3.OperationalError:
39 print("Waiting for lock on build directory")
40 con = sqlite3.Connection(path, timeout=0.25)
44 curs.execute("BEGIN EXCLUSIVE")
46 except sqlite3.OperationalError:
51 print("warning: sqlite3 not available in python, skipping build directory lock")
52 print("please file an issue on rust-lang/rust")
53 print("this is not a problem for non-concurrent x.py invocations")
58 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
59 temp_path = temp_file.name
60 with tarfile.open(temp_path, "w:xz"):
63 except tarfile.CompressionError:
66 def get(base, url, path, checksums, verbose=False, do_verify=True, help_on_error=None):
67 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
68 temp_path = temp_file.name
72 if url not in checksums:
73 raise RuntimeError("src/stage0.json doesn't contain a checksum for {}".format(url))
74 sha256 = checksums[url]
75 if os.path.exists(path):
76 if verify(path, sha256, False):
78 print("using already-download file", path)
82 print("ignoring already-download file",
83 path, "due to failed verification")
85 download(temp_path, "{}/{}".format(base, url), True, verbose, help_on_error=help_on_error)
86 if do_verify and not verify(temp_path, sha256, verbose):
87 raise RuntimeError("failed verification")
89 print("moving {} to {}".format(temp_path, path))
90 shutil.move(temp_path, path)
92 if os.path.isfile(temp_path):
94 print("removing", temp_path)
98 def download(path, url, probably_big, verbose, help_on_error=None):
101 _download(path, url, probably_big, verbose, True, help_on_error=help_on_error)
104 print("\nspurious failure, trying again")
105 _download(path, url, probably_big, verbose, False, help_on_error=help_on_error)
108 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
109 if probably_big or verbose:
110 print("downloading {}".format(url))
111 # see https://serverfault.com/questions/301128/how-to-download
112 if sys.platform == 'win32':
113 run(["PowerShell.exe", "/nologo", "-Command",
114 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
115 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
119 if probably_big or verbose:
123 require(["curl", "--version"])
125 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
126 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
127 "--retry", "3", "-Sf", "-o", path, url],
130 help_on_error=help_on_error)
133 def verify(path, expected, verbose):
134 """Check if the sha256 sum of the given path is valid"""
136 print("verifying", path)
137 with open(path, "rb") as source:
138 found = hashlib.sha256(source.read()).hexdigest()
139 verified = found == expected
141 print("invalid checksum:\n"
143 " expected: {}".format(found, expected))
147 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
148 """Unpack the given tarball file"""
149 print("extracting", tarball)
150 fname = os.path.basename(tarball).replace(tarball_suffix, "")
151 with contextlib.closing(tarfile.open(tarball)) as tar:
152 for member in tar.getnames():
153 if "/" not in member:
155 name = member.replace(fname + "/", "", 1)
156 if match is not None and not name.startswith(match):
158 name = name[len(match) + 1:]
160 dst_path = os.path.join(dst, name)
162 print(" extracting", member)
163 tar.extract(member, dst)
164 src_path = os.path.join(dst, member)
165 if os.path.isdir(src_path) and os.path.exists(dst_path):
167 shutil.move(src_path, dst_path)
168 shutil.rmtree(os.path.join(dst, fname))
171 def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=None, **kwargs):
172 """Run a child program in a new process"""
174 print("running: " + ' '.join(args))
176 # Use Popen here instead of call() as it apparently allows powershell on
177 # Windows to not lock up waiting for input presumably.
178 ret = subprocess.Popen(args, **kwargs)
181 err = "failed to run: " + ' '.join(args)
182 if help_on_error is not None:
183 err += "\n" + help_on_error
184 if verbose or exception:
185 raise RuntimeError(err)
186 # For most failures, we definitely do want to print this error, or the user will have no
187 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
188 # have already printed an error above, so there's no need to print the exact command we're
196 def require(cmd, exit=True):
197 '''Run a command, returning its output.
199 If `exit` is `True`, exit the process.
200 Otherwise, return None.'''
202 return subprocess.check_output(cmd).strip()
203 except (subprocess.CalledProcessError, OSError) as exc:
206 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
207 print("Please make sure it's installed and in the path.")
211 def format_build_time(duration):
212 """Return a nicer format for build time
214 >>> format_build_time('300')
217 return str(datetime.timedelta(seconds=int(duration)))
220 def default_build_triple(verbose):
221 """Build triple as in LLVM"""
222 # If the user already has a host build triple with an existing `rustc`
223 # install, use their preference. This fixes most issues with Windows builds
224 # being detected as GNU instead of MSVC.
225 default_encoding = sys.getdefaultencoding()
227 version = subprocess.check_output(["rustc", "--version", "--verbose"],
228 stderr=subprocess.DEVNULL)
229 version = version.decode(default_encoding)
230 host = next(x for x in version.split('\n') if x.startswith("host: "))
231 triple = host.split("host: ")[1]
233 print("detected default triple {} from pre-installed rustc".format(triple))
235 except Exception as e:
237 print("pre-installed rustc not detected: {}".format(e))
238 print("falling back to auto-detect")
240 required = sys.platform != 'win32'
241 ostype = require(["uname", "-s"], exit=required)
242 cputype = require(['uname', '-m'], exit=required)
244 # If we do not have `uname`, assume Windows.
245 if ostype is None or cputype is None:
246 return 'x86_64-pc-windows-msvc'
248 ostype = ostype.decode(default_encoding)
249 cputype = cputype.decode(default_encoding)
251 # The goal here is to come up with the same triple as LLVM would,
252 # at least for the subset of platforms we're willing to target.
254 'Darwin': 'apple-darwin',
255 'DragonFly': 'unknown-dragonfly',
256 'FreeBSD': 'unknown-freebsd',
257 'Haiku': 'unknown-haiku',
258 'NetBSD': 'unknown-netbsd',
259 'OpenBSD': 'unknown-openbsd'
262 # Consider the direct transformation first and then the special cases
263 if ostype in ostype_mapper:
264 ostype = ostype_mapper[ostype]
265 elif ostype == 'Linux':
266 os_from_sp = subprocess.check_output(
267 ['uname', '-o']).strip().decode(default_encoding)
268 if os_from_sp == 'Android':
269 ostype = 'linux-android'
271 ostype = 'unknown-linux-gnu'
272 elif ostype == 'SunOS':
273 ostype = 'pc-solaris'
274 # On Solaris, uname -m will return a machine classification instead
275 # of a cpu type, so uname -p is recommended instead. However, the
276 # output from that option is too generic for our purposes (it will
277 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
278 # must be used instead.
279 cputype = require(['isainfo', '-k']).decode(default_encoding)
280 # sparc cpus have sun as a target vendor
281 if 'sparc' in cputype:
282 ostype = 'sun-solaris'
283 elif ostype.startswith('MINGW'):
284 # msys' `uname` does not print gcc configuration, but prints msys
285 # configuration. so we cannot believe `uname -m`:
286 # msys1 is always i686 and msys2 is always x86_64.
287 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
289 ostype = 'pc-windows-gnu'
291 if os.environ.get('MSYSTEM') == 'MINGW64':
293 elif ostype.startswith('MSYS'):
294 ostype = 'pc-windows-gnu'
295 elif ostype.startswith('CYGWIN_NT'):
297 if ostype.endswith('WOW64'):
299 ostype = 'pc-windows-gnu'
300 elif sys.platform == 'win32':
301 # Some Windows platforms might have a `uname` command that returns a
302 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
303 # these cases, fall back to using sys.platform.
304 return 'x86_64-pc-windows-msvc'
306 err = "unknown OS type: {}".format(ostype)
309 if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
310 cputype = subprocess.check_output(
311 ['uname', '-p']).strip().decode(default_encoding)
314 'aarch64': 'aarch64',
322 'powerpc': 'powerpc',
323 'powerpc64': 'powerpc64',
324 'powerpc64le': 'powerpc64le',
326 'ppc64': 'powerpc64',
327 'ppc64le': 'powerpc64le',
328 'riscv64': 'riscv64gc',
336 # Consider the direct transformation first and then the special cases
337 if cputype in cputype_mapper:
338 cputype = cputype_mapper[cputype]
339 elif cputype in {'xscale', 'arm'}:
341 if ostype == 'linux-android':
342 ostype = 'linux-androideabi'
343 elif ostype == 'unknown-freebsd':
344 cputype = subprocess.check_output(
345 ['uname', '-p']).strip().decode(default_encoding)
346 ostype = 'unknown-freebsd'
347 elif cputype == 'armv6l':
349 if ostype == 'linux-android':
350 ostype = 'linux-androideabi'
353 elif cputype in {'armv7l', 'armv8l'}:
355 if ostype == 'linux-android':
356 ostype = 'linux-androideabi'
359 elif cputype == 'mips':
360 if sys.byteorder == 'big':
362 elif sys.byteorder == 'little':
365 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
366 elif cputype == 'mips64':
367 if sys.byteorder == 'big':
369 elif sys.byteorder == 'little':
372 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
373 # only the n64 ABI is supported, indicate it
375 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
378 err = "unknown cpu type: {}".format(cputype)
381 return "{}-{}".format(cputype, ostype)
384 @contextlib.contextmanager
385 def output(filepath):
386 tmp = filepath + '.tmp'
387 with open(tmp, 'w') as f:
390 if os.path.exists(filepath):
391 os.remove(filepath) # PermissionError/OSError on Win32 if in use
393 shutil.copy2(tmp, filepath)
396 os.rename(tmp, filepath)
399 class Stage0Toolchain:
400 def __init__(self, stage0_payload):
401 self.date = stage0_payload["date"]
402 self.version = stage0_payload["version"]
405 return self.version + "-" + self.date
408 class RustBuild(object):
409 """Provide all the methods required to build Rust"""
411 self.checksums_sha256 = {}
412 self.stage0_compiler = None
413 self.stage0_rustfmt = None
414 self._download_url = ''
418 self.config_toml = ''
420 self.use_locked_deps = ''
421 self.use_vendored_sources = ''
423 self.git_version = None
424 self.nix_deps_dir = None
425 self.rustc_commit = None
427 def download_toolchain(self, stage0=True, rustc_channel=None):
428 """Fetch the build system for Rust, written in Rust
430 This method will build a cache directory, then it will fetch the
431 tarball which has the stage0 compiler used to then bootstrap the Rust
434 Each downloaded tarball is extracted, after that, the script
435 will move all the content to the right place.
437 if rustc_channel is None:
438 rustc_channel = self.stage0_compiler.version
439 bin_root = self.bin_root(stage0)
441 key = self.stage0_compiler.date
443 key += str(self.rustc_commit)
444 if self.rustc(stage0).startswith(bin_root) and \
445 (not os.path.exists(self.rustc(stage0)) or
446 self.program_out_of_date(self.rustc_stamp(stage0), key)):
447 if os.path.exists(bin_root):
448 shutil.rmtree(bin_root)
449 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
450 filename = "rust-std-{}-{}{}".format(
451 rustc_channel, self.build, tarball_suffix)
452 pattern = "rust-std-{}".format(self.build)
453 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
454 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
456 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
457 # download-rustc doesn't need its own cargo, it can just use beta's.
459 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
461 self._download_component_helper(filename, "cargo", tarball_suffix)
462 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
464 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
465 self._download_component_helper(
466 filename, "rustc-dev", tarball_suffix, stage0
469 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
470 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
471 lib_dir = "{}/lib".format(bin_root)
472 for lib in os.listdir(lib_dir):
473 if lib.endswith(".so"):
474 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
475 with output(self.rustc_stamp(stage0)) as rust_stamp:
476 rust_stamp.write(key)
478 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
479 not os.path.exists(self.rustfmt())
480 or self.program_out_of_date(
481 self.rustfmt_stamp(),
482 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
485 if self.stage0_rustfmt is not None:
486 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
487 filename = "rustfmt-{}-{}{}".format(
488 self.stage0_rustfmt.version, self.build, tarball_suffix,
490 self._download_component_helper(
491 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
493 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
494 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
495 with output(self.rustfmt_stamp()) as rustfmt_stamp:
496 rustfmt_stamp.write(self.stage0_rustfmt.channel())
498 # Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
499 if self.downloading_llvm() and stage0:
500 # We want the most recent LLVM submodule update to avoid downloading
501 # LLVM more often than necessary.
503 # This git command finds that commit SHA, looking for bors-authored
504 # commits that modified src/llvm-project or other relevant version
507 # This works even in a repository that has not yet initialized
509 top_level = subprocess.check_output([
510 "git", "rev-parse", "--show-toplevel",
511 ]).decode(sys.getdefaultencoding()).strip()
512 llvm_sha = subprocess.check_output([
513 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
514 "--first-parent", "HEAD",
516 "{}/src/llvm-project".format(top_level),
517 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
518 # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
519 "{}/src/version".format(top_level)
520 ]).decode(sys.getdefaultencoding()).strip()
521 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
522 llvm_root = self.llvm_root()
523 llvm_lib = os.path.join(llvm_root, "lib")
524 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
525 self._download_ci_llvm(llvm_sha, llvm_assertions)
526 for binary in ["llvm-config", "FileCheck"]:
527 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
528 for lib in os.listdir(llvm_lib):
529 if lib.endswith(".so"):
530 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
531 with output(self.llvm_stamp()) as llvm_stamp:
532 llvm_stamp.write(llvm_sha + str(llvm_assertions))
534 def downloading_llvm(self):
535 opt = self.get_toml('download-ci-llvm', 'llvm')
536 # This is currently all tier 1 targets and tier 2 targets with host tools
537 # (since others may not have CI artifacts)
538 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
539 supported_platforms = [
541 "aarch64-unknown-linux-gnu",
542 "i686-pc-windows-gnu",
543 "i686-pc-windows-msvc",
544 "i686-unknown-linux-gnu",
545 "x86_64-unknown-linux-gnu",
546 "x86_64-apple-darwin",
547 "x86_64-pc-windows-gnu",
548 "x86_64-pc-windows-msvc",
549 # tier 2 with host tools
550 "aarch64-apple-darwin",
551 "aarch64-pc-windows-msvc",
552 "aarch64-unknown-linux-musl",
553 "arm-unknown-linux-gnueabi",
554 "arm-unknown-linux-gnueabihf",
555 "armv7-unknown-linux-gnueabihf",
556 "mips-unknown-linux-gnu",
557 "mips64-unknown-linux-gnuabi64",
558 "mips64el-unknown-linux-gnuabi64",
559 "mipsel-unknown-linux-gnu",
560 "powerpc-unknown-linux-gnu",
561 "powerpc64-unknown-linux-gnu",
562 "powerpc64le-unknown-linux-gnu",
563 "riscv64gc-unknown-linux-gnu",
564 "s390x-unknown-linux-gnu",
565 "x86_64-unknown-freebsd",
566 "x86_64-unknown-illumos",
567 "x86_64-unknown-linux-musl",
568 "x86_64-unknown-netbsd",
570 return opt == "true" \
571 or (opt == "if-available" and self.build in supported_platforms)
573 def _download_component_helper(
574 self, filename, pattern, tarball_suffix, stage0=True, key=None
578 key = self.stage0_compiler.date
580 key = self.rustc_commit
581 cache_dst = os.path.join(self.build_dir, "cache")
582 rustc_cache = os.path.join(cache_dst, key)
583 if not os.path.exists(rustc_cache):
584 os.makedirs(rustc_cache)
587 base = self._download_url
588 url = "dist/{}".format(key)
590 base = "https://ci-artifacts.rust-lang.org"
591 url = "rustc-builds/{}".format(self.rustc_commit)
592 tarball = os.path.join(rustc_cache, filename)
593 if not os.path.exists(tarball):
596 "{}/{}".format(url, filename),
598 self.checksums_sha256,
599 verbose=self.verbose,
602 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
604 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
606 print("error: could not find commit hash for downloading LLVM")
607 print("help: maybe your repository history is too shallow?")
608 print("help: consider disabling `download-ci-llvm`")
609 print("help: or fetch enough history to include one upstream commit")
611 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
612 cache_dst = os.path.join(self.build_dir, "cache")
613 rustc_cache = os.path.join(cache_dst, cache_prefix)
614 if not os.path.exists(rustc_cache):
615 os.makedirs(rustc_cache)
617 base = "https://ci-artifacts.rust-lang.org"
618 url = "rustc-builds/{}".format(llvm_sha)
620 url = url.replace('rustc-builds', 'rustc-builds-alt')
621 # ci-artifacts are only stored as .xz, not .gz
623 print("error: XZ support is required to download LLVM")
624 print("help: consider disabling `download-ci-llvm` or using python3")
626 tarball_suffix = '.tar.xz'
627 filename = "rust-dev-nightly-" + self.build + tarball_suffix
628 tarball = os.path.join(rustc_cache, filename)
629 if not os.path.exists(tarball):
630 help_on_error = "error: failed to download llvm from ci"
631 help_on_error += "\nhelp: old builds get deleted after a certain time"
632 help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
633 help_on_error += " disable `download-ci-llvm` in config.toml:"
634 help_on_error += "\n"
635 help_on_error += "\n[llvm]"
636 help_on_error += "\ndownload-ci-llvm = false"
637 help_on_error += "\n"
640 "{}/{}".format(url, filename),
642 self.checksums_sha256,
643 verbose=self.verbose,
645 help_on_error=help_on_error,
647 unpack(tarball, tarball_suffix, self.llvm_root(),
649 verbose=self.verbose)
651 def fix_bin_or_dylib(self, fname):
652 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
653 or the RPATH section, to fix the dynamic library search path
655 This method is only required on NixOS and uses the PatchELF utility to
656 change the interpreter/RPATH of ELF executables.
658 Please see https://nixos.org/patchelf.html for more information
660 default_encoding = sys.getdefaultencoding()
662 ostype = subprocess.check_output(
663 ['uname', '-s']).strip().decode(default_encoding)
664 except subprocess.CalledProcessError:
666 except OSError as reason:
667 if getattr(reason, 'winerror', None) is not None:
671 if ostype != "Linux":
674 # If the user has asked binaries to be patched for Nix, then
675 # don't check for NixOS or `/lib`, just continue to the patching.
676 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
677 # Use `/etc/os-release` instead of `/etc/NIXOS`.
678 # The latter one does not exist on NixOS when using tmpfs as root.
680 with open("/etc/os-release", "r") as f:
681 if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
683 except FileNotFoundError:
685 if os.path.exists("/lib"):
688 # At this point we're pretty sure the user is running NixOS or
690 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
691 print(nix_os_msg, fname)
693 # Only build `.nix-deps` once.
694 nix_deps_dir = self.nix_deps_dir
696 # Run `nix-build` to "build" each dependency (which will likely reuse
697 # the existing `/nix/store` copy, or at most download a pre-built copy).
699 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
700 # directory, but still reference the actual `/nix/store` path in the rpath
701 # as it makes it significantly more robust against changes to the location of
702 # the `.nix-deps` location.
704 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
705 # zlib: Needed as a system dependency of `libLLVM-*.so`.
706 # patchelf: Needed for patching ELF binaries (see doc comment above).
707 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
709 with (import <nixpkgs> {});
711 name = "rust-stage0-dependencies";
720 subprocess.check_output([
721 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
723 except subprocess.CalledProcessError as reason:
724 print("warning: failed to call nix-build:", reason)
726 self.nix_deps_dir = nix_deps_dir
728 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
730 # Relative default, all binary and dynamic libraries we ship
731 # appear to have this (even when `../lib` is redundant).
733 os.path.join(os.path.realpath(nix_deps_dir), "lib")
735 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
736 if not fname.endswith(".so"):
737 # Finally, set the corret .interp for binaries
738 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
739 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
742 subprocess.check_output([patchelf] + patchelf_args + [fname])
743 except subprocess.CalledProcessError as reason:
744 print("warning: failed to call patchelf:", reason)
747 # If `download-rustc` is set, download the most recent commit with CI artifacts
748 def maybe_download_ci_toolchain(self):
749 # If `download-rustc` is not set, default to rebuilding.
750 download_rustc = self.get_toml("download-rustc", section="rust")
751 if download_rustc is None or download_rustc == "false":
753 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
755 # Handle running from a directory other than the top level
756 rev_parse = ["git", "rev-parse", "--show-toplevel"]
757 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
758 compiler = "{}/compiler/".format(top_level)
759 library = "{}/library/".format(top_level)
761 # Look for a version to compare to based on the current commit.
762 # Only commits merged by bors will have CI artifacts.
764 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
765 "--first-parent", "HEAD"
767 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
769 print("error: could not find commit hash for downloading rustc")
770 print("help: maybe your repository history is too shallow?")
771 print("help: consider disabling `download-rustc`")
772 print("help: or fetch enough history to include one upstream commit")
775 # Warn if there were changes to the compiler or standard library since the ancestor commit.
776 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
778 if download_rustc == "if-unchanged":
780 print("warning: saw changes to compiler/ or library/ since {}; " \
781 "ignoring `download-rustc`".format(commit))
783 print("warning: `download-rustc` is enabled, but there are changes to " \
784 "compiler/ or library/")
787 print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
788 self.rustc_commit = commit
789 # FIXME: support downloading artifacts from the beta channel
790 self.download_toolchain(False, "nightly")
792 def rustc_stamp(self, stage0):
793 """Return the path for .rustc-stamp at the given stage
796 >>> rb.build_dir = "build"
797 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
799 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
802 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
804 def rustfmt_stamp(self):
805 """Return the path for .rustfmt-stamp
808 >>> rb.build_dir = "build"
809 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
812 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
814 def llvm_stamp(self):
815 """Return the path for .rustfmt-stamp
818 >>> rb.build_dir = "build"
819 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
822 return os.path.join(self.llvm_root(), '.llvm-stamp')
825 def program_out_of_date(self, stamp_path, key):
826 """Check if the given program stamp is out of date"""
827 if not os.path.exists(stamp_path) or self.clean:
829 with open(stamp_path, 'r') as stamp:
830 return key != stamp.read()
832 def bin_root(self, stage0):
833 """Return the binary root directory for the given stage
836 >>> rb.build_dir = "build"
837 >>> rb.bin_root(True) == os.path.join("build", "stage0")
839 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
842 When the 'build' property is given should be a nested directory:
844 >>> rb.build = "devel"
845 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
852 return os.path.join(self.build_dir, self.build, subdir)
855 """Return the CI LLVM root directory
858 >>> rb.build_dir = "build"
859 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
862 When the 'build' property is given should be a nested directory:
864 >>> rb.build = "devel"
865 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
868 return os.path.join(self.build_dir, self.build, "ci-llvm")
870 def get_toml(self, key, section=None):
871 """Returns the value of the given key in config.toml, otherwise returns None
874 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
875 >>> rb.get_toml("key2")
878 If the key does not exist, the result is None:
880 >>> rb.get_toml("key3") is None
883 Optionally also matches the section the key appears in
885 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
886 >>> rb.get_toml('key', 'a')
888 >>> rb.get_toml('key', 'b')
890 >>> rb.get_toml('key', 'c') is None
893 >>> rb.config_toml = 'key1 = true'
894 >>> rb.get_toml("key1")
899 for line in self.config_toml.splitlines():
900 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
901 if section_match is not None:
902 cur_section = section_match.group(1)
904 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
905 if match is not None:
906 value = match.group(1)
907 if section is None or section == cur_section:
908 return self.get_string(value) or value.strip()
912 """Return config path for cargo"""
913 return self.program_config('cargo')
915 def rustc(self, stage0):
916 """Return config path for rustc"""
917 return self.program_config('rustc', stage0)
920 """Return config path for rustfmt"""
921 if self.stage0_rustfmt is None:
923 return self.program_config('rustfmt')
925 def program_config(self, program, stage0=True):
926 """Return config path for the given program at the given stage
929 >>> rb.config_toml = 'rustc = "rustc"\\n'
930 >>> rb.program_config('rustc')
932 >>> rb.config_toml = ''
933 >>> cargo_path = rb.program_config('cargo', True)
934 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
937 >>> cargo_path = rb.program_config('cargo', False)
938 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
942 config = self.get_toml(program)
944 return os.path.expanduser(config)
945 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
946 program, self.exe_suffix()))
949 def get_string(line):
950 """Return the value between double quotes
952 >>> RustBuild.get_string(' "devel" ')
954 >>> RustBuild.get_string(" 'devel' ")
956 >>> RustBuild.get_string('devel') is None
958 >>> RustBuild.get_string(' "devel ')
961 start = line.find('"')
963 end = start + 1 + line[start + 1:].find('"')
964 return line[start + 1:end]
965 start = line.find('\'')
967 end = start + 1 + line[start + 1:].find('\'')
968 return line[start + 1:end]
973 """Return a suffix for executables"""
974 if sys.platform == 'win32':
978 def bootstrap_binary(self):
979 """Return the path of the bootstrap binary
982 >>> rb.build_dir = "build"
983 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
984 ... "debug", "bootstrap")
987 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
989 def build_bootstrap(self):
990 """Build bootstrap"""
991 print("Building rustbuild")
992 build_dir = os.path.join(self.build_dir, "bootstrap")
993 if self.clean and os.path.exists(build_dir):
994 shutil.rmtree(build_dir)
995 env = os.environ.copy()
996 # `CARGO_BUILD_TARGET` breaks bootstrap build.
997 # See also: <https://github.com/rust-lang/rust/issues/70208>.
998 if "CARGO_BUILD_TARGET" in env:
999 del env["CARGO_BUILD_TARGET"]
1000 env["CARGO_TARGET_DIR"] = build_dir
1001 env["RUSTC"] = self.rustc(True)
1002 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1003 (os.pathsep + env["LD_LIBRARY_PATH"]) \
1004 if "LD_LIBRARY_PATH" in env else ""
1005 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1006 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
1007 if "DYLD_LIBRARY_PATH" in env else ""
1008 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
1009 (os.pathsep + env["LIBRARY_PATH"]) \
1010 if "LIBRARY_PATH" in env else ""
1012 # preserve existing RUSTFLAGS
1013 env.setdefault("RUSTFLAGS", "")
1014 build_section = "target.{}".format(self.build)
1015 target_features = []
1016 if self.get_toml("crt-static", build_section) == "true":
1017 target_features += ["+crt-static"]
1018 elif self.get_toml("crt-static", build_section) == "false":
1019 target_features += ["-crt-static"]
1021 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
1022 target_linker = self.get_toml("linker", build_section)
1023 if target_linker is not None:
1024 env["RUSTFLAGS"] += " -C linker=" + target_linker
1025 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
1026 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
1027 if self.get_toml("deny-warnings", "rust") != "false":
1028 env["RUSTFLAGS"] += " -Dwarnings"
1030 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
1031 os.pathsep + env["PATH"]
1032 if not os.path.isfile(self.cargo()):
1033 raise Exception("no cargo executable found at `{}`".format(
1035 args = [self.cargo(), "build", "--manifest-path",
1036 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
1037 for _ in range(0, self.verbose):
1038 args.append("--verbose")
1039 if self.use_locked_deps:
1040 args.append("--locked")
1041 if self.use_vendored_sources:
1042 args.append("--frozen")
1043 run(args, env=env, verbose=self.verbose)
1045 def build_triple(self):
1046 """Build triple as in LLVM
1048 Note that `default_build_triple` is moderately expensive,
1049 so use `self.build` where possible.
1051 config = self.get_toml('build')
1054 return default_build_triple(self.verbose)
1056 def check_submodule(self, module, slow_submodules):
1057 if not slow_submodules:
1058 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
1059 cwd=os.path.join(self.rust_root, module),
1060 stdout=subprocess.PIPE)
1065 def update_submodule(self, module, checked_out, recorded_submodules):
1066 module_path = os.path.join(self.rust_root, module)
1068 if checked_out is not None:
1069 default_encoding = sys.getdefaultencoding()
1070 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
1071 if recorded_submodules[module] == checked_out:
1074 print("Updating submodule", module)
1076 run(["git", "submodule", "-q", "sync", module],
1077 cwd=self.rust_root, verbose=self.verbose)
1079 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
1080 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
1081 update_args.append("--progress")
1082 update_args.append(module)
1084 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
1085 except RuntimeError:
1086 print("Failed updating submodule. This is probably due to uncommitted local changes.")
1087 print('Either stash the changes by running "git stash" within the submodule\'s')
1088 print('directory, reset them by running "git reset --hard", or commit them.')
1089 print("To reset all submodules' changes run", end=" ")
1090 print('"git submodule foreach --recursive git reset --hard".')
1093 run(["git", "reset", "-q", "--hard"],
1094 cwd=module_path, verbose=self.verbose)
1095 run(["git", "clean", "-qdfx"],
1096 cwd=module_path, verbose=self.verbose)
1098 def update_submodules(self):
1099 """Update submodules"""
1100 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
1101 self.get_toml('submodules') == "false":
1104 default_encoding = sys.getdefaultencoding()
1106 # check the existence and version of 'git' command
1107 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
1108 self.git_version = distutils.version.LooseVersion(git_version_str)
1110 slow_submodules = self.get_toml('fast-submodules') == "false"
1113 print('Unconditionally updating submodules')
1115 print('Updating only changed submodules')
1116 default_encoding = sys.getdefaultencoding()
1117 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
1118 # currently requires everything in a workspace to be "locally present" when starting a
1119 # build, and will give a hard error if any Cargo.toml files are missing.
1120 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
1121 # share a workspace with any tools - maybe it could be excluded from the workspace?
1122 # That will still require cloning the submodules the second you check the standard
1123 # library, though...
1124 # FIXME: Is there a way to avoid hard-coding the submodules required?
1125 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1127 "src/tools/rust-installer",
1131 "library/backtrace",
1134 filtered_submodules = []
1135 submodules_names = []
1136 for module in submodules:
1137 check = self.check_submodule(module, slow_submodules)
1138 filtered_submodules.append((module, check))
1139 submodules_names.append(module)
1140 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1141 cwd=self.rust_root, stdout=subprocess.PIPE)
1142 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1143 # { filename: hash }
1144 recorded_submodules = {}
1145 for data in recorded:
1146 # [mode, kind, hash, filename]
1148 recorded_submodules[data[3]] = data[2]
1149 for module in filtered_submodules:
1150 self.update_submodule(module[0], module[1], recorded_submodules)
1151 print(" Submodules updated in %.2f seconds" % (time() - start_time))
1153 def set_dist_environment(self, url):
1154 """Set download URL for normal environment"""
1155 if 'RUSTUP_DIST_SERVER' in os.environ:
1156 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1158 self._download_url = url
1160 def check_vendored_status(self):
1161 """Check that vendoring is configured properly"""
1162 vendor_dir = os.path.join(self.rust_root, 'vendor')
1163 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1164 if os.environ.get('USER') != os.environ['SUDO_USER']:
1165 self.use_vendored_sources = True
1166 print('info: looks like you are running this command under `sudo`')
1167 print(' and so in order to preserve your $HOME this will now')
1168 print(' use vendored sources by default.')
1169 if not os.path.exists(vendor_dir):
1170 print('error: vendoring required, but vendor directory does not exist.')
1171 print(' Run `cargo vendor` without sudo to initialize the '
1172 'vendor directory.')
1173 raise Exception("{} not found".format(vendor_dir))
1175 if self.use_vendored_sources:
1176 config = ("[source.crates-io]\n"
1177 "replace-with = 'vendored-sources'\n"
1178 "registry = 'https://example.com'\n"
1180 "[source.vendored-sources]\n"
1181 "directory = '{}/vendor'\n"
1182 .format(self.rust_root))
1183 if not os.path.exists('.cargo'):
1184 os.makedirs('.cargo')
1185 with output('.cargo/config') as cargo_config:
1186 cargo_config.write(config)
1188 print('info: using vendored source, but .cargo/config is already present.')
1189 print(' Reusing the current configuration file. But you may want to '
1190 'configure vendoring like this:')
1193 if os.path.exists('.cargo'):
1194 shutil.rmtree('.cargo')
1196 def ensure_vendored(self):
1197 """Ensure that the vendored sources are available if needed"""
1198 vendor_dir = os.path.join(self.rust_root, 'vendor')
1199 # Note that this does not handle updating the vendored dependencies if
1200 # the rust git repository is updated. Normal development usually does
1201 # not use vendoring, so hopefully this isn't too much of a problem.
1202 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1206 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1207 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1208 ], verbose=self.verbose, cwd=self.rust_root)
1211 def bootstrap(help_triggered):
1212 """Configure, fetch, build and run the initial bootstrap"""
1214 # If the user is asking for help, let them know that the whole download-and-build
1215 # process has to happen before anything is printed out.
1217 print("info: Downloading and building bootstrap before processing --help")
1218 print(" command. See src/bootstrap/README.md for help with common")
1221 parser = argparse.ArgumentParser(description='Build rust')
1222 parser.add_argument('--config')
1223 parser.add_argument('--build')
1224 parser.add_argument('--clean', action='store_true')
1225 parser.add_argument('-v', '--verbose', action='count', default=0)
1227 args = [a for a in sys.argv if a != '-h' and a != '--help']
1228 args, _ = parser.parse_known_args(args)
1230 # Configure initial bootstrap
1232 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1233 build.verbose = args.verbose
1234 build.clean = args.clean
1236 # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1237 # then `config.toml` in the root directory.
1238 toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1239 using_default_path = toml_path is None
1240 if using_default_path:
1241 toml_path = 'config.toml'
1242 if not os.path.exists(toml_path):
1243 toml_path = os.path.join(build.rust_root, toml_path)
1245 # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1246 # but not if `config.toml` hasn't been created.
1247 if not using_default_path or os.path.exists(toml_path):
1248 with open(toml_path) as config:
1249 build.config_toml = config.read()
1251 profile = build.get_toml('profile')
1252 if profile is not None:
1253 include_file = 'config.{}.toml'.format(profile)
1254 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1255 include_path = os.path.join(include_dir, include_file)
1256 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1257 # specific key, so appending our defaults at the end allows the user to override them
1258 with open(include_path) as included_toml:
1259 build.config_toml += os.linesep + included_toml.read()
1261 config_verbose = build.get_toml('verbose', 'build')
1262 if config_verbose is not None:
1263 build.verbose = max(build.verbose, int(config_verbose))
1265 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1267 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1269 build.check_vendored_status()
1271 build_dir = build.get_toml('build-dir', 'build') or 'build'
1272 build.build_dir = os.path.abspath(build_dir)
1274 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1276 build.checksums_sha256 = data["checksums_sha256"]
1277 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1278 if data.get("rustfmt") is not None:
1279 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1281 build.set_dist_environment(data["dist_server"])
1283 build.build = args.build or build.build_triple()
1285 # Acquire the lock before doing any build actions
1286 # The lock is released when `lock` is dropped
1287 if not os.path.exists(build.build_dir):
1288 os.makedirs(build.build_dir)
1289 lock = acquire_lock(build.build_dir)
1290 build.update_submodules()
1292 # Fetch/build the bootstrap
1293 build.download_toolchain()
1294 # Download the master compiler if `download-rustc` is set
1295 build.maybe_download_ci_toolchain()
1297 build.ensure_vendored()
1298 build.build_bootstrap()
1302 args = [build.bootstrap_binary()]
1303 args.extend(sys.argv[1:])
1304 env = os.environ.copy()
1305 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1306 env["BOOTSTRAP_PYTHON"] = sys.executable
1307 env["RUSTC_BOOTSTRAP"] = '1'
1308 if build.rustc_commit is not None:
1309 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1310 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1314 """Entry point for the bootstrap process"""
1317 # x.py help <cmd> ...
1318 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1319 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1322 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1324 bootstrap(help_triggered)
1325 if not help_triggered:
1326 print("Build completed successfully in {}".format(
1327 format_build_time(time() - start_time)))
1328 except (SystemExit, KeyboardInterrupt) as error:
1329 if hasattr(error, 'code') and isinstance(error.code, int):
1330 exit_code = error.code
1334 if not help_triggered:
1335 print("Build completed unsuccessfully in {}".format(
1336 format_build_time(time() - start_time)))
1340 if __name__ == '__main__':