1 from __future__ import absolute_import, division, print_function
5 import distutils.version
16 from time import time, sleep
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
25 path = os.path.join(build_dir, "lock.db")
27 con = sqlite3.Connection(path, timeout=0)
29 curs.execute("BEGIN EXCLUSIVE")
30 # The lock is released when the cursor is dropped
32 # If the database is busy then lock has already been acquired
33 # so we wait for the lock.
34 # We retry every quarter second so that execution is passed back to python
35 # so that it can handle signals
36 except sqlite3.OperationalError:
39 print("Waiting for lock on build directory")
40 con = sqlite3.Connection(path, timeout=0.25)
44 curs.execute("BEGIN EXCLUSIVE")
46 except sqlite3.OperationalError:
51 print("warning: sqlite3 not available in python, skipping build directory lock")
52 print("please file an issue on rust-lang/rust")
53 print("this is not a problem for non-concurrent x.py invocations")
58 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
59 temp_path = temp_file.name
60 with tarfile.open(temp_path, "w:xz"):
63 except tarfile.CompressionError:
66 def get(base, url, path, checksums, verbose=False, do_verify=True, help_on_error=None):
67 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
68 temp_path = temp_file.name
72 if url not in checksums:
73 raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
74 "Pre-built artifacts might not available for this "
75 "target at this time, see https://doc.rust-lang.org/nightly"
76 "/rustc/platform-support.html for more information.")
78 sha256 = checksums[url]
79 if os.path.exists(path):
80 if verify(path, sha256, False):
82 print("using already-download file", path)
86 print("ignoring already-download file",
87 path, "due to failed verification")
89 download(temp_path, "{}/{}".format(base, url), True, verbose, help_on_error=help_on_error)
90 if do_verify and not verify(temp_path, sha256, verbose):
91 raise RuntimeError("failed verification")
93 print("moving {} to {}".format(temp_path, path))
94 shutil.move(temp_path, path)
96 if os.path.isfile(temp_path):
98 print("removing", temp_path)
102 def download(path, url, probably_big, verbose, help_on_error=None):
103 for _ in range(0, 4):
105 _download(path, url, probably_big, verbose, True, help_on_error=help_on_error)
108 print("\nspurious failure, trying again")
109 _download(path, url, probably_big, verbose, False, help_on_error=help_on_error)
112 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
113 # Try to use curl (potentially available on win32
114 # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/)
115 # If an error occurs:
116 # - If we are on win32 fallback to powershell
117 # - Otherwise raise the error if appropriate
118 if probably_big or verbose:
119 print("downloading {}".format(url))
121 platform_is_win32 = sys.platform == 'win32'
123 if probably_big or verbose:
127 # If curl is not present on Win32, we shoud not sys.exit
128 # but raise `CalledProcessError` or `OSError` instead
129 require(["curl", "--version"], exception=platform_is_win32)
131 "-L", # Follow redirect.
132 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
133 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
134 "--retry", "3", "-Sf", "-o", path, url],
136 exception=True, # Will raise RuntimeError on failure
137 help_on_error=help_on_error)
138 except (subprocess.CalledProcessError, OSError, RuntimeError):
139 # see http://serverfault.com/questions/301128/how-to-download
140 if platform_is_win32:
141 run(["PowerShell.exe", "/nologo", "-Command",
142 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
143 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
146 # Check if the RuntimeError raised by run(curl) should be silenced
147 elif verbose or exception:
151 def verify(path, expected, verbose):
152 """Check if the sha256 sum of the given path is valid"""
154 print("verifying", path)
155 with open(path, "rb") as source:
156 found = hashlib.sha256(source.read()).hexdigest()
157 verified = found == expected
159 print("invalid checksum:\n"
161 " expected: {}".format(found, expected))
165 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
166 """Unpack the given tarball file"""
167 print("extracting", tarball)
168 fname = os.path.basename(tarball).replace(tarball_suffix, "")
169 with contextlib.closing(tarfile.open(tarball)) as tar:
170 for member in tar.getnames():
171 if "/" not in member:
173 name = member.replace(fname + "/", "", 1)
174 if match is not None and not name.startswith(match):
176 name = name[len(match) + 1:]
178 dst_path = os.path.join(dst, name)
180 print(" extracting", member)
181 tar.extract(member, dst)
182 src_path = os.path.join(dst, member)
183 if os.path.isdir(src_path) and os.path.exists(dst_path):
185 shutil.move(src_path, dst_path)
186 shutil.rmtree(os.path.join(dst, fname))
189 def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=None, **kwargs):
190 """Run a child program in a new process"""
192 print("running: " + ' '.join(args))
194 # Use Popen here instead of call() as it apparently allows powershell on
195 # Windows to not lock up waiting for input presumably.
196 ret = subprocess.Popen(args, **kwargs)
199 err = "failed to run: " + ' '.join(args)
200 if help_on_error is not None:
201 err += "\n" + help_on_error
202 if verbose or exception:
203 raise RuntimeError(err)
204 # For most failures, we definitely do want to print this error, or the user will have no
205 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
206 # have already printed an error above, so there's no need to print the exact command we're
214 def require(cmd, exit=True, exception=False):
215 '''Run a command, returning its output.
217 If `exception` is `True`, raise the error
218 Otherwise If `exit` is `True`, exit the process
221 return subprocess.check_output(cmd).strip()
222 except (subprocess.CalledProcessError, OSError) as exc:
226 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
227 print("Please make sure it's installed and in the path.")
233 def format_build_time(duration):
234 """Return a nicer format for build time
236 >>> format_build_time('300')
239 return str(datetime.timedelta(seconds=int(duration)))
242 def default_build_triple(verbose):
243 """Build triple as in LLVM"""
244 # If the user already has a host build triple with an existing `rustc`
245 # install, use their preference. This fixes most issues with Windows builds
246 # being detected as GNU instead of MSVC.
247 default_encoding = sys.getdefaultencoding()
249 version = subprocess.check_output(["rustc", "--version", "--verbose"],
250 stderr=subprocess.DEVNULL)
251 version = version.decode(default_encoding)
252 host = next(x for x in version.split('\n') if x.startswith("host: "))
253 triple = host.split("host: ")[1]
255 print("detected default triple {} from pre-installed rustc".format(triple))
257 except Exception as e:
259 print("pre-installed rustc not detected: {}".format(e))
260 print("falling back to auto-detect")
262 required = sys.platform != 'win32'
263 ostype = require(["uname", "-s"], exit=required)
264 cputype = require(['uname', '-m'], exit=required)
266 # If we do not have `uname`, assume Windows.
267 if ostype is None or cputype is None:
268 return 'x86_64-pc-windows-msvc'
270 ostype = ostype.decode(default_encoding)
271 cputype = cputype.decode(default_encoding)
273 # The goal here is to come up with the same triple as LLVM would,
274 # at least for the subset of platforms we're willing to target.
276 'Darwin': 'apple-darwin',
277 'DragonFly': 'unknown-dragonfly',
278 'FreeBSD': 'unknown-freebsd',
279 'Haiku': 'unknown-haiku',
280 'NetBSD': 'unknown-netbsd',
281 'OpenBSD': 'unknown-openbsd'
284 # Consider the direct transformation first and then the special cases
285 if ostype in ostype_mapper:
286 ostype = ostype_mapper[ostype]
287 elif ostype == 'Linux':
288 os_from_sp = subprocess.check_output(
289 ['uname', '-o']).strip().decode(default_encoding)
290 if os_from_sp == 'Android':
291 ostype = 'linux-android'
293 ostype = 'unknown-linux-gnu'
294 elif ostype == 'SunOS':
295 ostype = 'pc-solaris'
296 # On Solaris, uname -m will return a machine classification instead
297 # of a cpu type, so uname -p is recommended instead. However, the
298 # output from that option is too generic for our purposes (it will
299 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
300 # must be used instead.
301 cputype = require(['isainfo', '-k']).decode(default_encoding)
302 # sparc cpus have sun as a target vendor
303 if 'sparc' in cputype:
304 ostype = 'sun-solaris'
305 elif ostype.startswith('MINGW'):
306 # msys' `uname` does not print gcc configuration, but prints msys
307 # configuration. so we cannot believe `uname -m`:
308 # msys1 is always i686 and msys2 is always x86_64.
309 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
311 ostype = 'pc-windows-gnu'
313 if os.environ.get('MSYSTEM') == 'MINGW64':
315 elif ostype.startswith('MSYS'):
316 ostype = 'pc-windows-gnu'
317 elif ostype.startswith('CYGWIN_NT'):
319 if ostype.endswith('WOW64'):
321 ostype = 'pc-windows-gnu'
322 elif sys.platform == 'win32':
323 # Some Windows platforms might have a `uname` command that returns a
324 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
325 # these cases, fall back to using sys.platform.
326 return 'x86_64-pc-windows-msvc'
328 err = "unknown OS type: {}".format(ostype)
331 if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
332 cputype = subprocess.check_output(
333 ['uname', '-p']).strip().decode(default_encoding)
336 'aarch64': 'aarch64',
344 'powerpc': 'powerpc',
345 'powerpc64': 'powerpc64',
346 'powerpc64le': 'powerpc64le',
348 'ppc64': 'powerpc64',
349 'ppc64le': 'powerpc64le',
350 'riscv64': 'riscv64gc',
358 # Consider the direct transformation first and then the special cases
359 if cputype in cputype_mapper:
360 cputype = cputype_mapper[cputype]
361 elif cputype in {'xscale', 'arm'}:
363 if ostype == 'linux-android':
364 ostype = 'linux-androideabi'
365 elif ostype == 'unknown-freebsd':
366 cputype = subprocess.check_output(
367 ['uname', '-p']).strip().decode(default_encoding)
368 ostype = 'unknown-freebsd'
369 elif cputype == 'armv6l':
371 if ostype == 'linux-android':
372 ostype = 'linux-androideabi'
375 elif cputype in {'armv7l', 'armv8l'}:
377 if ostype == 'linux-android':
378 ostype = 'linux-androideabi'
381 elif cputype == 'mips':
382 if sys.byteorder == 'big':
384 elif sys.byteorder == 'little':
387 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
388 elif cputype == 'mips64':
389 if sys.byteorder == 'big':
391 elif sys.byteorder == 'little':
394 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
395 # only the n64 ABI is supported, indicate it
397 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
400 err = "unknown cpu type: {}".format(cputype)
403 return "{}-{}".format(cputype, ostype)
406 @contextlib.contextmanager
407 def output(filepath):
408 tmp = filepath + '.tmp'
409 with open(tmp, 'w') as f:
412 if os.path.exists(filepath):
413 os.remove(filepath) # PermissionError/OSError on Win32 if in use
415 shutil.copy2(tmp, filepath)
418 os.rename(tmp, filepath)
421 class Stage0Toolchain:
422 def __init__(self, stage0_payload):
423 self.date = stage0_payload["date"]
424 self.version = stage0_payload["version"]
427 return self.version + "-" + self.date
430 class RustBuild(object):
431 """Provide all the methods required to build Rust"""
433 self.checksums_sha256 = {}
434 self.stage0_compiler = None
435 self.stage0_rustfmt = None
436 self._download_url = ''
440 self.config_toml = ''
442 self.use_locked_deps = ''
443 self.use_vendored_sources = ''
445 self.git_version = None
446 self.nix_deps_dir = None
447 self.rustc_commit = None
449 def download_toolchain(self, stage0=True, rustc_channel=None):
450 """Fetch the build system for Rust, written in Rust
452 This method will build a cache directory, then it will fetch the
453 tarball which has the stage0 compiler used to then bootstrap the Rust
456 Each downloaded tarball is extracted, after that, the script
457 will move all the content to the right place.
459 if rustc_channel is None:
460 rustc_channel = self.stage0_compiler.version
461 bin_root = self.bin_root(stage0)
463 key = self.stage0_compiler.date
465 key += str(self.rustc_commit)
466 if self.rustc(stage0).startswith(bin_root) and \
467 (not os.path.exists(self.rustc(stage0)) or
468 self.program_out_of_date(self.rustc_stamp(stage0), key)):
469 if os.path.exists(bin_root):
470 shutil.rmtree(bin_root)
471 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
472 filename = "rust-std-{}-{}{}".format(
473 rustc_channel, self.build, tarball_suffix)
474 pattern = "rust-std-{}".format(self.build)
475 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
476 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
478 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
479 # download-rustc doesn't need its own cargo, it can just use beta's.
481 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
483 self._download_component_helper(filename, "cargo", tarball_suffix)
484 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
486 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
487 self._download_component_helper(
488 filename, "rustc-dev", tarball_suffix, stage0
491 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
492 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
493 lib_dir = "{}/lib".format(bin_root)
494 for lib in os.listdir(lib_dir):
495 if lib.endswith(".so"):
496 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
497 with output(self.rustc_stamp(stage0)) as rust_stamp:
498 rust_stamp.write(key)
500 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
501 not os.path.exists(self.rustfmt())
502 or self.program_out_of_date(
503 self.rustfmt_stamp(),
504 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
507 if self.stage0_rustfmt is not None:
508 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
509 filename = "rustfmt-{}-{}{}".format(
510 self.stage0_rustfmt.version, self.build, tarball_suffix,
512 self._download_component_helper(
513 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
515 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
516 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
517 with output(self.rustfmt_stamp()) as rustfmt_stamp:
518 rustfmt_stamp.write(self.stage0_rustfmt.channel())
520 def _download_component_helper(
521 self, filename, pattern, tarball_suffix, stage0=True, key=None
525 key = self.stage0_compiler.date
527 key = self.rustc_commit
528 cache_dst = os.path.join(self.build_dir, "cache")
529 rustc_cache = os.path.join(cache_dst, key)
530 if not os.path.exists(rustc_cache):
531 os.makedirs(rustc_cache)
534 base = self._download_url
535 url = "dist/{}".format(key)
537 base = "https://ci-artifacts.rust-lang.org"
538 url = "rustc-builds/{}".format(self.rustc_commit)
539 tarball = os.path.join(rustc_cache, filename)
540 if not os.path.exists(tarball):
543 "{}/{}".format(url, filename),
545 self.checksums_sha256,
546 verbose=self.verbose,
549 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
551 def fix_bin_or_dylib(self, fname):
552 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
553 or the RPATH section, to fix the dynamic library search path
555 This method is only required on NixOS and uses the PatchELF utility to
556 change the interpreter/RPATH of ELF executables.
558 Please see https://nixos.org/patchelf.html for more information
560 default_encoding = sys.getdefaultencoding()
562 ostype = subprocess.check_output(
563 ['uname', '-s']).strip().decode(default_encoding)
564 except subprocess.CalledProcessError:
566 except OSError as reason:
567 if getattr(reason, 'winerror', None) is not None:
571 if ostype != "Linux":
574 # If the user has asked binaries to be patched for Nix, then
575 # don't check for NixOS or `/lib`, just continue to the patching.
576 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
577 # Use `/etc/os-release` instead of `/etc/NIXOS`.
578 # The latter one does not exist on NixOS when using tmpfs as root.
580 with open("/etc/os-release", "r") as f:
581 if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
583 except FileNotFoundError:
585 if os.path.exists("/lib"):
588 # At this point we're pretty sure the user is running NixOS or
590 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
591 print(nix_os_msg, fname)
593 # Only build `.nix-deps` once.
594 nix_deps_dir = self.nix_deps_dir
596 # Run `nix-build` to "build" each dependency (which will likely reuse
597 # the existing `/nix/store` copy, or at most download a pre-built copy).
599 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
600 # directory, but still reference the actual `/nix/store` path in the rpath
601 # as it makes it significantly more robust against changes to the location of
602 # the `.nix-deps` location.
604 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
605 # zlib: Needed as a system dependency of `libLLVM-*.so`.
606 # patchelf: Needed for patching ELF binaries (see doc comment above).
607 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
609 with (import <nixpkgs> {});
611 name = "rust-stage0-dependencies";
620 subprocess.check_output([
621 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
623 except subprocess.CalledProcessError as reason:
624 print("warning: failed to call nix-build:", reason)
626 self.nix_deps_dir = nix_deps_dir
628 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
630 # Relative default, all binary and dynamic libraries we ship
631 # appear to have this (even when `../lib` is redundant).
633 os.path.join(os.path.realpath(nix_deps_dir), "lib")
635 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
636 if not fname.endswith(".so"):
637 # Finally, set the corret .interp for binaries
638 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
639 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
642 subprocess.check_output([patchelf] + patchelf_args + [fname])
643 except subprocess.CalledProcessError as reason:
644 print("warning: failed to call patchelf:", reason)
647 # If `download-rustc` is set, download the most recent commit with CI artifacts
648 def maybe_download_ci_toolchain(self):
649 # If `download-rustc` is not set, default to rebuilding.
650 download_rustc = self.get_toml("download-rustc", section="rust")
651 if download_rustc is None or download_rustc == "false":
653 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
655 # Handle running from a directory other than the top level
656 rev_parse = ["git", "rev-parse", "--show-toplevel"]
657 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
658 compiler = "{}/compiler/".format(top_level)
659 library = "{}/library/".format(top_level)
661 # Look for a version to compare to based on the current commit.
662 # Only commits merged by bors will have CI artifacts.
664 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
665 "--first-parent", "HEAD"
667 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
669 print("error: could not find commit hash for downloading rustc")
670 print("help: maybe your repository history is too shallow?")
671 print("help: consider disabling `download-rustc`")
672 print("help: or fetch enough history to include one upstream commit")
675 # Warn if there were changes to the compiler or standard library since the ancestor commit.
676 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
678 if download_rustc == "if-unchanged":
680 print("warning: saw changes to compiler/ or library/ since {}; " \
681 "ignoring `download-rustc`".format(commit))
683 print("warning: `download-rustc` is enabled, but there are changes to " \
684 "compiler/ or library/")
687 print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
688 self.rustc_commit = commit
689 # FIXME: support downloading artifacts from the beta channel
690 self.download_toolchain(False, "nightly")
692 def rustc_stamp(self, stage0):
693 """Return the path for .rustc-stamp at the given stage
696 >>> rb.build_dir = "build"
697 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
699 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
702 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
704 def rustfmt_stamp(self):
705 """Return the path for .rustfmt-stamp
708 >>> rb.build_dir = "build"
709 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
712 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
714 def program_out_of_date(self, stamp_path, key):
715 """Check if the given program stamp is out of date"""
716 if not os.path.exists(stamp_path) or self.clean:
718 with open(stamp_path, 'r') as stamp:
719 return key != stamp.read()
721 def bin_root(self, stage0):
722 """Return the binary root directory for the given stage
725 >>> rb.build_dir = "build"
726 >>> rb.bin_root(True) == os.path.join("build", "stage0")
728 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
731 When the 'build' property is given should be a nested directory:
733 >>> rb.build = "devel"
734 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
741 return os.path.join(self.build_dir, self.build, subdir)
743 def get_toml(self, key, section=None):
744 """Returns the value of the given key in config.toml, otherwise returns None
747 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
748 >>> rb.get_toml("key2")
751 If the key does not exist, the result is None:
753 >>> rb.get_toml("key3") is None
756 Optionally also matches the section the key appears in
758 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
759 >>> rb.get_toml('key', 'a')
761 >>> rb.get_toml('key', 'b')
763 >>> rb.get_toml('key', 'c') is None
766 >>> rb.config_toml = 'key1 = true'
767 >>> rb.get_toml("key1")
772 for line in self.config_toml.splitlines():
773 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
774 if section_match is not None:
775 cur_section = section_match.group(1)
777 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
778 if match is not None:
779 value = match.group(1)
780 if section is None or section == cur_section:
781 return self.get_string(value) or value.strip()
785 """Return config path for cargo"""
786 return self.program_config('cargo')
788 def rustc(self, stage0):
789 """Return config path for rustc"""
790 return self.program_config('rustc', stage0)
793 """Return config path for rustfmt"""
794 if self.stage0_rustfmt is None:
796 return self.program_config('rustfmt')
798 def program_config(self, program, stage0=True):
799 """Return config path for the given program at the given stage
802 >>> rb.config_toml = 'rustc = "rustc"\\n'
803 >>> rb.program_config('rustc')
805 >>> rb.config_toml = ''
806 >>> cargo_path = rb.program_config('cargo', True)
807 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
810 >>> cargo_path = rb.program_config('cargo', False)
811 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
815 config = self.get_toml(program)
817 return os.path.expanduser(config)
818 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
819 program, self.exe_suffix()))
822 def get_string(line):
823 """Return the value between double quotes
825 >>> RustBuild.get_string(' "devel" ')
827 >>> RustBuild.get_string(" 'devel' ")
829 >>> RustBuild.get_string('devel') is None
831 >>> RustBuild.get_string(' "devel ')
834 start = line.find('"')
836 end = start + 1 + line[start + 1:].find('"')
837 return line[start + 1:end]
838 start = line.find('\'')
840 end = start + 1 + line[start + 1:].find('\'')
841 return line[start + 1:end]
846 """Return a suffix for executables"""
847 if sys.platform == 'win32':
851 def bootstrap_binary(self):
852 """Return the path of the bootstrap binary
855 >>> rb.build_dir = "build"
856 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
857 ... "debug", "bootstrap")
860 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
862 def build_bootstrap(self):
863 """Build bootstrap"""
864 print("Building rustbuild")
865 build_dir = os.path.join(self.build_dir, "bootstrap")
866 if self.clean and os.path.exists(build_dir):
867 shutil.rmtree(build_dir)
868 env = os.environ.copy()
869 # `CARGO_BUILD_TARGET` breaks bootstrap build.
870 # See also: <https://github.com/rust-lang/rust/issues/70208>.
871 if "CARGO_BUILD_TARGET" in env:
872 del env["CARGO_BUILD_TARGET"]
873 env["CARGO_TARGET_DIR"] = build_dir
874 env["RUSTC"] = self.rustc(True)
875 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
876 (os.pathsep + env["LD_LIBRARY_PATH"]) \
877 if "LD_LIBRARY_PATH" in env else ""
878 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
879 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
880 if "DYLD_LIBRARY_PATH" in env else ""
881 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
882 (os.pathsep + env["LIBRARY_PATH"]) \
883 if "LIBRARY_PATH" in env else ""
885 # preserve existing RUSTFLAGS
886 env.setdefault("RUSTFLAGS", "")
887 build_section = "target.{}".format(self.build)
889 if self.get_toml("crt-static", build_section) == "true":
890 target_features += ["+crt-static"]
891 elif self.get_toml("crt-static", build_section) == "false":
892 target_features += ["-crt-static"]
894 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
895 target_linker = self.get_toml("linker", build_section)
896 if target_linker is not None:
897 env["RUSTFLAGS"] += " -C linker=" + target_linker
898 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
899 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
900 if self.get_toml("deny-warnings", "rust") != "false":
901 env["RUSTFLAGS"] += " -Dwarnings"
903 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
904 os.pathsep + env["PATH"]
905 if not os.path.isfile(self.cargo()):
906 raise Exception("no cargo executable found at `{}`".format(
908 args = [self.cargo(), "build", "--manifest-path",
909 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
910 for _ in range(0, self.verbose):
911 args.append("--verbose")
912 if self.use_locked_deps:
913 args.append("--locked")
914 if self.use_vendored_sources:
915 args.append("--frozen")
916 run(args, env=env, verbose=self.verbose)
918 def build_triple(self):
919 """Build triple as in LLVM
921 Note that `default_build_triple` is moderately expensive,
922 so use `self.build` where possible.
924 config = self.get_toml('build')
927 return default_build_triple(self.verbose)
929 def check_submodule(self, module):
930 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
931 cwd=os.path.join(self.rust_root, module),
932 stdout=subprocess.PIPE)
935 def update_submodule(self, module, checked_out, recorded_submodules):
936 module_path = os.path.join(self.rust_root, module)
938 default_encoding = sys.getdefaultencoding()
939 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
940 if recorded_submodules[module] == checked_out:
943 print("Updating submodule", module)
945 run(["git", "submodule", "-q", "sync", module],
946 cwd=self.rust_root, verbose=self.verbose)
948 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
949 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
950 update_args.append("--progress")
951 update_args.append(module)
953 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
955 print("Failed updating submodule. This is probably due to uncommitted local changes.")
956 print('Either stash the changes by running "git stash" within the submodule\'s')
957 print('directory, reset them by running "git reset --hard", or commit them.')
958 print("To reset all submodules' changes run", end=" ")
959 print('"git submodule foreach --recursive git reset --hard".')
962 run(["git", "reset", "-q", "--hard"],
963 cwd=module_path, verbose=self.verbose)
964 run(["git", "clean", "-qdfx"],
965 cwd=module_path, verbose=self.verbose)
967 def update_submodules(self):
968 """Update submodules"""
969 has_git = os.path.exists(os.path.join(self.rust_root, ".git"))
970 # This just arbitrarily checks for cargo, but any workspace member in
971 # a submodule would work.
972 has_submodules = os.path.exists(os.path.join(self.rust_root, "src/tools/cargo/Cargo.toml"))
973 if not has_git and not has_submodules:
974 print("This is not a git repository, and the requisite git submodules were not found.")
975 print("If you downloaded the source from https://github.com/rust-lang/rust/releases,")
976 print("those sources will not work. Instead, consider downloading from the source")
977 print("releases linked at")
978 print("https://forge.rust-lang.org/infra/other-installation-methods.html#source-code")
979 print("or clone the repository at https://github.com/rust-lang/rust/.")
981 if not has_git or self.get_toml('submodules') == "false":
984 default_encoding = sys.getdefaultencoding()
986 # check the existence and version of 'git' command
987 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
988 self.git_version = distutils.version.LooseVersion(git_version_str)
991 print('Updating only changed submodules')
992 default_encoding = sys.getdefaultencoding()
993 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
994 # currently requires everything in a workspace to be "locally present" when starting a
995 # build, and will give a hard error if any Cargo.toml files are missing.
996 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
997 # share a workspace with any tools - maybe it could be excluded from the workspace?
998 # That will still require cloning the submodules the second you check the standard
1000 # FIXME: Is there a way to avoid hard-coding the submodules required?
1001 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
1003 "src/tools/rust-installer",
1007 "library/backtrace",
1010 # If build.vendor is set in config.toml, we must update rust-analyzer also.
1011 # Otherwise, the bootstrap will fail (#96456).
1012 if self.use_vendored_sources:
1013 submodules.append("src/tools/rust-analyzer")
1014 filtered_submodules = []
1015 submodules_names = []
1016 for module in submodules:
1017 check = self.check_submodule(module)
1018 filtered_submodules.append((module, check))
1019 submodules_names.append(module)
1020 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1021 cwd=self.rust_root, stdout=subprocess.PIPE)
1022 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1023 # { filename: hash }
1024 recorded_submodules = {}
1025 for data in recorded:
1026 # [mode, kind, hash, filename]
1028 recorded_submodules[data[3]] = data[2]
1029 for module in filtered_submodules:
1030 self.update_submodule(module[0], module[1], recorded_submodules)
1031 print(" Submodules updated in %.2f seconds" % (time() - start_time))
1033 def set_dist_environment(self, url):
1034 """Set download URL for normal environment"""
1035 if 'RUSTUP_DIST_SERVER' in os.environ:
1036 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1038 self._download_url = url
1040 def check_vendored_status(self):
1041 """Check that vendoring is configured properly"""
1042 vendor_dir = os.path.join(self.rust_root, 'vendor')
1043 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1044 if os.getuid() == 0:
1045 self.use_vendored_sources = True
1046 print('info: looks like you\'re trying to run this command as root')
1047 print(' and so in order to preserve your $HOME this will now')
1048 print(' use vendored sources by default.')
1049 if not os.path.exists(vendor_dir):
1050 print('error: vendoring required, but vendor directory does not exist.')
1051 print(' Run `cargo vendor` without sudo to initialize the '
1052 'vendor directory.')
1053 raise Exception("{} not found".format(vendor_dir))
1055 if self.use_vendored_sources:
1056 config = ("[source.crates-io]\n"
1057 "replace-with = 'vendored-sources'\n"
1058 "registry = 'https://example.com'\n"
1060 "[source.vendored-sources]\n"
1061 "directory = '{}/vendor'\n"
1062 .format(self.rust_root))
1063 if not os.path.exists('.cargo'):
1064 os.makedirs('.cargo')
1065 with output('.cargo/config') as cargo_config:
1066 cargo_config.write(config)
1068 print('info: using vendored source, but .cargo/config is already present.')
1069 print(' Reusing the current configuration file. But you may want to '
1070 'configure vendoring like this:')
1073 if os.path.exists('.cargo'):
1074 shutil.rmtree('.cargo')
1076 def ensure_vendored(self):
1077 """Ensure that the vendored sources are available if needed"""
1078 vendor_dir = os.path.join(self.rust_root, 'vendor')
1079 # Note that this does not handle updating the vendored dependencies if
1080 # the rust git repository is updated. Normal development usually does
1081 # not use vendoring, so hopefully this isn't too much of a problem.
1082 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1086 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1087 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1088 ], verbose=self.verbose, cwd=self.rust_root)
1091 def bootstrap(help_triggered):
1092 """Configure, fetch, build and run the initial bootstrap"""
1094 # If the user is asking for help, let them know that the whole download-and-build
1095 # process has to happen before anything is printed out.
1097 print("info: Downloading and building bootstrap before processing --help")
1098 print(" command. See src/bootstrap/README.md for help with common")
1101 parser = argparse.ArgumentParser(description='Build rust')
1102 parser.add_argument('--config')
1103 parser.add_argument('--build')
1104 parser.add_argument('--clean', action='store_true')
1105 parser.add_argument('-v', '--verbose', action='count', default=0)
1107 args = [a for a in sys.argv if a != '-h' and a != '--help']
1108 args, _ = parser.parse_known_args(args)
1110 # Configure initial bootstrap
1112 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1113 build.verbose = args.verbose
1114 build.clean = args.clean
1116 # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1117 # then `config.toml` in the root directory.
1118 toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1119 using_default_path = toml_path is None
1120 if using_default_path:
1121 toml_path = 'config.toml'
1122 if not os.path.exists(toml_path):
1123 toml_path = os.path.join(build.rust_root, toml_path)
1125 # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1126 # but not if `config.toml` hasn't been created.
1127 if not using_default_path or os.path.exists(toml_path):
1128 with open(toml_path) as config:
1129 build.config_toml = config.read()
1131 profile = build.get_toml('profile')
1132 if profile is not None:
1133 include_file = 'config.{}.toml'.format(profile)
1134 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1135 include_path = os.path.join(include_dir, include_file)
1136 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1137 # specific key, so appending our defaults at the end allows the user to override them
1138 with open(include_path) as included_toml:
1139 build.config_toml += os.linesep + included_toml.read()
1141 config_verbose = build.get_toml('verbose', 'build')
1142 if config_verbose is not None:
1143 build.verbose = max(build.verbose, int(config_verbose))
1145 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1147 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1149 build.check_vendored_status()
1151 build_dir = build.get_toml('build-dir', 'build') or 'build'
1152 build.build_dir = os.path.abspath(build_dir)
1154 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1156 build.checksums_sha256 = data["checksums_sha256"]
1157 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1158 if data.get("rustfmt") is not None:
1159 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1161 build.set_dist_environment(data["dist_server"])
1163 build.build = args.build or build.build_triple()
1165 # Acquire the lock before doing any build actions
1166 # The lock is released when `lock` is dropped
1167 if not os.path.exists(build.build_dir):
1168 os.makedirs(build.build_dir)
1169 lock = acquire_lock(build.build_dir)
1170 build.update_submodules()
1172 # Fetch/build the bootstrap
1173 build.download_toolchain()
1174 # Download the master compiler if `download-rustc` is set
1175 build.maybe_download_ci_toolchain()
1177 build.ensure_vendored()
1178 build.build_bootstrap()
1182 args = [build.bootstrap_binary()]
1183 args.extend(sys.argv[1:])
1184 env = os.environ.copy()
1185 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1186 env["BOOTSTRAP_PYTHON"] = sys.executable
1187 if build.rustc_commit is not None:
1188 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1189 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1193 """Entry point for the bootstrap process"""
1196 # x.py help <cmd> ...
1197 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1198 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1201 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1203 bootstrap(help_triggered)
1204 if not help_triggered:
1205 print("Build completed successfully in {}".format(
1206 format_build_time(time() - start_time)))
1207 except (SystemExit, KeyboardInterrupt) as error:
1208 if hasattr(error, 'code') and isinstance(error.code, int):
1209 exit_code = error.code
1213 if not help_triggered:
1214 print("Build completed unsuccessfully in {}".format(
1215 format_build_time(time() - start_time)))
1219 if __name__ == '__main__':