1 from __future__ import absolute_import, division, print_function
5 import distutils.version
16 from time import time, sleep
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
25 path = os.path.join(build_dir, "lock.db")
27 con = sqlite3.Connection(path, timeout=0)
29 curs.execute("BEGIN EXCLUSIVE")
30 # The lock is released when the cursor is dropped
32 # If the database is busy then lock has already been acquired
33 # so we wait for the lock.
34 # We retry every quarter second so that execution is passed back to python
35 # so that it can handle signals
36 except sqlite3.OperationalError:
39 print("Waiting for lock on build directory")
40 con = sqlite3.Connection(path, timeout=0.25)
44 curs.execute("BEGIN EXCLUSIVE")
46 except sqlite3.OperationalError:
51 print("warning: sqlite3 not available in python, skipping build directory lock")
52 print("please file an issue on rust-lang/rust")
53 print("this is not a problem for non-concurrent x.py invocations")
58 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
59 temp_path = temp_file.name
60 with tarfile.open(temp_path, "w:xz"):
63 except tarfile.CompressionError:
66 def get(base, url, path, checksums, verbose=False, do_verify=True, help_on_error=None):
67 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
68 temp_path = temp_file.name
72 if url not in checksums:
73 raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
74 "Pre-built artifacts might not available for this "
75 "target at this time, see https://doc.rust-lang.org/nightly"
76 "/rustc/platform-support.html for more information.")
78 sha256 = checksums[url]
79 if os.path.exists(path):
80 if verify(path, sha256, False):
82 print("using already-download file", path)
86 print("ignoring already-download file",
87 path, "due to failed verification")
89 download(temp_path, "{}/{}".format(base, url), True, verbose, help_on_error=help_on_error)
90 if do_verify and not verify(temp_path, sha256, verbose):
91 raise RuntimeError("failed verification")
93 print("moving {} to {}".format(temp_path, path))
94 shutil.move(temp_path, path)
96 if os.path.isfile(temp_path):
98 print("removing", temp_path)
102 def download(path, url, probably_big, verbose, help_on_error=None):
103 for _ in range(0, 4):
105 _download(path, url, probably_big, verbose, True, help_on_error=help_on_error)
108 print("\nspurious failure, trying again")
109 _download(path, url, probably_big, verbose, False, help_on_error=help_on_error)
112 def _download(path, url, probably_big, verbose, exception, help_on_error=None):
113 if probably_big or verbose:
114 print("downloading {}".format(url))
115 # see https://serverfault.com/questions/301128/how-to-download
116 if sys.platform == 'win32':
117 run(["PowerShell.exe", "/nologo", "-Command",
118 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
119 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
123 if probably_big or verbose:
127 require(["curl", "--version"])
129 "-L", # Follow redirect.
130 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
131 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
132 "--retry", "3", "-Sf", "-o", path, url],
135 help_on_error=help_on_error)
138 def verify(path, expected, verbose):
139 """Check if the sha256 sum of the given path is valid"""
141 print("verifying", path)
142 with open(path, "rb") as source:
143 found = hashlib.sha256(source.read()).hexdigest()
144 verified = found == expected
146 print("invalid checksum:\n"
148 " expected: {}".format(found, expected))
152 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
153 """Unpack the given tarball file"""
154 print("extracting", tarball)
155 fname = os.path.basename(tarball).replace(tarball_suffix, "")
156 with contextlib.closing(tarfile.open(tarball)) as tar:
157 for member in tar.getnames():
158 if "/" not in member:
160 name = member.replace(fname + "/", "", 1)
161 if match is not None and not name.startswith(match):
163 name = name[len(match) + 1:]
165 dst_path = os.path.join(dst, name)
167 print(" extracting", member)
168 tar.extract(member, dst)
169 src_path = os.path.join(dst, member)
170 if os.path.isdir(src_path) and os.path.exists(dst_path):
172 shutil.move(src_path, dst_path)
173 shutil.rmtree(os.path.join(dst, fname))
176 def run(args, verbose=False, exception=False, is_bootstrap=False, help_on_error=None, **kwargs):
177 """Run a child program in a new process"""
179 print("running: " + ' '.join(args))
181 # Use Popen here instead of call() as it apparently allows powershell on
182 # Windows to not lock up waiting for input presumably.
183 ret = subprocess.Popen(args, **kwargs)
186 err = "failed to run: " + ' '.join(args)
187 if help_on_error is not None:
188 err += "\n" + help_on_error
189 if verbose or exception:
190 raise RuntimeError(err)
191 # For most failures, we definitely do want to print this error, or the user will have no
192 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
193 # have already printed an error above, so there's no need to print the exact command we're
201 def require(cmd, exit=True):
202 '''Run a command, returning its output.
204 If `exit` is `True`, exit the process.
205 Otherwise, return None.'''
207 return subprocess.check_output(cmd).strip()
208 except (subprocess.CalledProcessError, OSError) as exc:
211 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
212 print("Please make sure it's installed and in the path.")
216 def format_build_time(duration):
217 """Return a nicer format for build time
219 >>> format_build_time('300')
222 return str(datetime.timedelta(seconds=int(duration)))
225 def default_build_triple(verbose):
226 """Build triple as in LLVM"""
227 # If the user already has a host build triple with an existing `rustc`
228 # install, use their preference. This fixes most issues with Windows builds
229 # being detected as GNU instead of MSVC.
230 default_encoding = sys.getdefaultencoding()
232 version = subprocess.check_output(["rustc", "--version", "--verbose"],
233 stderr=subprocess.DEVNULL)
234 version = version.decode(default_encoding)
235 host = next(x for x in version.split('\n') if x.startswith("host: "))
236 triple = host.split("host: ")[1]
238 print("detected default triple {} from pre-installed rustc".format(triple))
240 except Exception as e:
242 print("pre-installed rustc not detected: {}".format(e))
243 print("falling back to auto-detect")
245 required = sys.platform != 'win32'
246 ostype = require(["uname", "-s"], exit=required)
247 cputype = require(['uname', '-m'], exit=required)
249 # If we do not have `uname`, assume Windows.
250 if ostype is None or cputype is None:
251 return 'x86_64-pc-windows-msvc'
253 ostype = ostype.decode(default_encoding)
254 cputype = cputype.decode(default_encoding)
256 # The goal here is to come up with the same triple as LLVM would,
257 # at least for the subset of platforms we're willing to target.
259 'Darwin': 'apple-darwin',
260 'DragonFly': 'unknown-dragonfly',
261 'FreeBSD': 'unknown-freebsd',
262 'Haiku': 'unknown-haiku',
263 'NetBSD': 'unknown-netbsd',
264 'OpenBSD': 'unknown-openbsd'
267 # Consider the direct transformation first and then the special cases
268 if ostype in ostype_mapper:
269 ostype = ostype_mapper[ostype]
270 elif ostype == 'Linux':
271 os_from_sp = subprocess.check_output(
272 ['uname', '-o']).strip().decode(default_encoding)
273 if os_from_sp == 'Android':
274 ostype = 'linux-android'
276 ostype = 'unknown-linux-gnu'
277 elif ostype == 'SunOS':
278 ostype = 'pc-solaris'
279 # On Solaris, uname -m will return a machine classification instead
280 # of a cpu type, so uname -p is recommended instead. However, the
281 # output from that option is too generic for our purposes (it will
282 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
283 # must be used instead.
284 cputype = require(['isainfo', '-k']).decode(default_encoding)
285 # sparc cpus have sun as a target vendor
286 if 'sparc' in cputype:
287 ostype = 'sun-solaris'
288 elif ostype.startswith('MINGW'):
289 # msys' `uname` does not print gcc configuration, but prints msys
290 # configuration. so we cannot believe `uname -m`:
291 # msys1 is always i686 and msys2 is always x86_64.
292 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
294 ostype = 'pc-windows-gnu'
296 if os.environ.get('MSYSTEM') == 'MINGW64':
298 elif ostype.startswith('MSYS'):
299 ostype = 'pc-windows-gnu'
300 elif ostype.startswith('CYGWIN_NT'):
302 if ostype.endswith('WOW64'):
304 ostype = 'pc-windows-gnu'
305 elif sys.platform == 'win32':
306 # Some Windows platforms might have a `uname` command that returns a
307 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
308 # these cases, fall back to using sys.platform.
309 return 'x86_64-pc-windows-msvc'
311 err = "unknown OS type: {}".format(ostype)
314 if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
315 cputype = subprocess.check_output(
316 ['uname', '-p']).strip().decode(default_encoding)
319 'aarch64': 'aarch64',
327 'powerpc': 'powerpc',
328 'powerpc64': 'powerpc64',
329 'powerpc64le': 'powerpc64le',
331 'ppc64': 'powerpc64',
332 'ppc64le': 'powerpc64le',
333 'riscv64': 'riscv64gc',
341 # Consider the direct transformation first and then the special cases
342 if cputype in cputype_mapper:
343 cputype = cputype_mapper[cputype]
344 elif cputype in {'xscale', 'arm'}:
346 if ostype == 'linux-android':
347 ostype = 'linux-androideabi'
348 elif ostype == 'unknown-freebsd':
349 cputype = subprocess.check_output(
350 ['uname', '-p']).strip().decode(default_encoding)
351 ostype = 'unknown-freebsd'
352 elif cputype == 'armv6l':
354 if ostype == 'linux-android':
355 ostype = 'linux-androideabi'
358 elif cputype in {'armv7l', 'armv8l'}:
360 if ostype == 'linux-android':
361 ostype = 'linux-androideabi'
364 elif cputype == 'mips':
365 if sys.byteorder == 'big':
367 elif sys.byteorder == 'little':
370 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
371 elif cputype == 'mips64':
372 if sys.byteorder == 'big':
374 elif sys.byteorder == 'little':
377 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
378 # only the n64 ABI is supported, indicate it
380 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
383 err = "unknown cpu type: {}".format(cputype)
386 return "{}-{}".format(cputype, ostype)
389 @contextlib.contextmanager
390 def output(filepath):
391 tmp = filepath + '.tmp'
392 with open(tmp, 'w') as f:
395 if os.path.exists(filepath):
396 os.remove(filepath) # PermissionError/OSError on Win32 if in use
398 shutil.copy2(tmp, filepath)
401 os.rename(tmp, filepath)
404 class Stage0Toolchain:
405 def __init__(self, stage0_payload):
406 self.date = stage0_payload["date"]
407 self.version = stage0_payload["version"]
410 return self.version + "-" + self.date
413 class RustBuild(object):
414 """Provide all the methods required to build Rust"""
416 self.checksums_sha256 = {}
417 self.stage0_compiler = None
418 self.stage0_rustfmt = None
419 self._download_url = ''
423 self.config_toml = ''
425 self.use_locked_deps = ''
426 self.use_vendored_sources = ''
428 self.git_version = None
429 self.nix_deps_dir = None
430 self.rustc_commit = None
432 def download_toolchain(self, stage0=True, rustc_channel=None):
433 """Fetch the build system for Rust, written in Rust
435 This method will build a cache directory, then it will fetch the
436 tarball which has the stage0 compiler used to then bootstrap the Rust
439 Each downloaded tarball is extracted, after that, the script
440 will move all the content to the right place.
442 if rustc_channel is None:
443 rustc_channel = self.stage0_compiler.version
444 bin_root = self.bin_root(stage0)
446 key = self.stage0_compiler.date
448 key += str(self.rustc_commit)
449 if self.rustc(stage0).startswith(bin_root) and \
450 (not os.path.exists(self.rustc(stage0)) or
451 self.program_out_of_date(self.rustc_stamp(stage0), key)):
452 if os.path.exists(bin_root):
453 shutil.rmtree(bin_root)
454 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
455 filename = "rust-std-{}-{}{}".format(
456 rustc_channel, self.build, tarball_suffix)
457 pattern = "rust-std-{}".format(self.build)
458 self._download_component_helper(filename, pattern, tarball_suffix, stage0)
459 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
461 self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
462 # download-rustc doesn't need its own cargo, it can just use beta's.
464 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
466 self._download_component_helper(filename, "cargo", tarball_suffix)
467 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
469 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
470 self._download_component_helper(
471 filename, "rustc-dev", tarball_suffix, stage0
474 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
475 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
476 lib_dir = "{}/lib".format(bin_root)
477 for lib in os.listdir(lib_dir):
478 if lib.endswith(".so"):
479 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
480 with output(self.rustc_stamp(stage0)) as rust_stamp:
481 rust_stamp.write(key)
483 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
484 not os.path.exists(self.rustfmt())
485 or self.program_out_of_date(
486 self.rustfmt_stamp(),
487 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
490 if self.stage0_rustfmt is not None:
491 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
492 filename = "rustfmt-{}-{}{}".format(
493 self.stage0_rustfmt.version, self.build, tarball_suffix,
495 self._download_component_helper(
496 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
498 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
499 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
500 with output(self.rustfmt_stamp()) as rustfmt_stamp:
501 rustfmt_stamp.write(self.stage0_rustfmt.channel())
503 def _download_component_helper(
504 self, filename, pattern, tarball_suffix, stage0=True, key=None
508 key = self.stage0_compiler.date
510 key = self.rustc_commit
511 cache_dst = os.path.join(self.build_dir, "cache")
512 rustc_cache = os.path.join(cache_dst, key)
513 if not os.path.exists(rustc_cache):
514 os.makedirs(rustc_cache)
517 base = self._download_url
518 url = "dist/{}".format(key)
520 base = "https://ci-artifacts.rust-lang.org"
521 url = "rustc-builds/{}".format(self.rustc_commit)
522 tarball = os.path.join(rustc_cache, filename)
523 if not os.path.exists(tarball):
526 "{}/{}".format(url, filename),
528 self.checksums_sha256,
529 verbose=self.verbose,
532 unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
534 def fix_bin_or_dylib(self, fname):
535 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
536 or the RPATH section, to fix the dynamic library search path
538 This method is only required on NixOS and uses the PatchELF utility to
539 change the interpreter/RPATH of ELF executables.
541 Please see https://nixos.org/patchelf.html for more information
543 default_encoding = sys.getdefaultencoding()
545 ostype = subprocess.check_output(
546 ['uname', '-s']).strip().decode(default_encoding)
547 except subprocess.CalledProcessError:
549 except OSError as reason:
550 if getattr(reason, 'winerror', None) is not None:
554 if ostype != "Linux":
557 # If the user has asked binaries to be patched for Nix, then
558 # don't check for NixOS or `/lib`, just continue to the patching.
559 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
560 # Use `/etc/os-release` instead of `/etc/NIXOS`.
561 # The latter one does not exist on NixOS when using tmpfs as root.
563 with open("/etc/os-release", "r") as f:
564 if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
566 except FileNotFoundError:
568 if os.path.exists("/lib"):
571 # At this point we're pretty sure the user is running NixOS or
573 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
574 print(nix_os_msg, fname)
576 # Only build `.nix-deps` once.
577 nix_deps_dir = self.nix_deps_dir
579 # Run `nix-build` to "build" each dependency (which will likely reuse
580 # the existing `/nix/store` copy, or at most download a pre-built copy).
582 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
583 # directory, but still reference the actual `/nix/store` path in the rpath
584 # as it makes it significantly more robust against changes to the location of
585 # the `.nix-deps` location.
587 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
588 # zlib: Needed as a system dependency of `libLLVM-*.so`.
589 # patchelf: Needed for patching ELF binaries (see doc comment above).
590 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
592 with (import <nixpkgs> {});
594 name = "rust-stage0-dependencies";
603 subprocess.check_output([
604 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
606 except subprocess.CalledProcessError as reason:
607 print("warning: failed to call nix-build:", reason)
609 self.nix_deps_dir = nix_deps_dir
611 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
613 # Relative default, all binary and dynamic libraries we ship
614 # appear to have this (even when `../lib` is redundant).
616 os.path.join(os.path.realpath(nix_deps_dir), "lib")
618 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
619 if not fname.endswith(".so"):
620 # Finally, set the corret .interp for binaries
621 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
622 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
625 subprocess.check_output([patchelf] + patchelf_args + [fname])
626 except subprocess.CalledProcessError as reason:
627 print("warning: failed to call patchelf:", reason)
630 # If `download-rustc` is set, download the most recent commit with CI artifacts
631 def maybe_download_ci_toolchain(self):
632 # If `download-rustc` is not set, default to rebuilding.
633 download_rustc = self.get_toml("download-rustc", section="rust")
634 if download_rustc is None or download_rustc == "false":
636 assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
638 # Handle running from a directory other than the top level
639 rev_parse = ["git", "rev-parse", "--show-toplevel"]
640 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
641 compiler = "{}/compiler/".format(top_level)
642 library = "{}/library/".format(top_level)
644 # Look for a version to compare to based on the current commit.
645 # Only commits merged by bors will have CI artifacts.
647 "git", "rev-list", "--author=bors@rust-lang.org", "-n1",
648 "--first-parent", "HEAD"
650 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
652 print("error: could not find commit hash for downloading rustc")
653 print("help: maybe your repository history is too shallow?")
654 print("help: consider disabling `download-rustc`")
655 print("help: or fetch enough history to include one upstream commit")
658 # Warn if there were changes to the compiler or standard library since the ancestor commit.
659 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
661 if download_rustc == "if-unchanged":
663 print("warning: saw changes to compiler/ or library/ since {}; " \
664 "ignoring `download-rustc`".format(commit))
666 print("warning: `download-rustc` is enabled, but there are changes to " \
667 "compiler/ or library/")
670 print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
671 self.rustc_commit = commit
672 # FIXME: support downloading artifacts from the beta channel
673 self.download_toolchain(False, "nightly")
675 def rustc_stamp(self, stage0):
676 """Return the path for .rustc-stamp at the given stage
679 >>> rb.build_dir = "build"
680 >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
682 >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
685 return os.path.join(self.bin_root(stage0), '.rustc-stamp')
687 def rustfmt_stamp(self):
688 """Return the path for .rustfmt-stamp
691 >>> rb.build_dir = "build"
692 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
695 return os.path.join(self.bin_root(True), '.rustfmt-stamp')
697 def program_out_of_date(self, stamp_path, key):
698 """Check if the given program stamp is out of date"""
699 if not os.path.exists(stamp_path) or self.clean:
701 with open(stamp_path, 'r') as stamp:
702 return key != stamp.read()
704 def bin_root(self, stage0):
705 """Return the binary root directory for the given stage
708 >>> rb.build_dir = "build"
709 >>> rb.bin_root(True) == os.path.join("build", "stage0")
711 >>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
714 When the 'build' property is given should be a nested directory:
716 >>> rb.build = "devel"
717 >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
724 return os.path.join(self.build_dir, self.build, subdir)
726 def get_toml(self, key, section=None):
727 """Returns the value of the given key in config.toml, otherwise returns None
730 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
731 >>> rb.get_toml("key2")
734 If the key does not exist, the result is None:
736 >>> rb.get_toml("key3") is None
739 Optionally also matches the section the key appears in
741 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
742 >>> rb.get_toml('key', 'a')
744 >>> rb.get_toml('key', 'b')
746 >>> rb.get_toml('key', 'c') is None
749 >>> rb.config_toml = 'key1 = true'
750 >>> rb.get_toml("key1")
755 for line in self.config_toml.splitlines():
756 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
757 if section_match is not None:
758 cur_section = section_match.group(1)
760 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
761 if match is not None:
762 value = match.group(1)
763 if section is None or section == cur_section:
764 return self.get_string(value) or value.strip()
768 """Return config path for cargo"""
769 return self.program_config('cargo')
771 def rustc(self, stage0):
772 """Return config path for rustc"""
773 return self.program_config('rustc', stage0)
776 """Return config path for rustfmt"""
777 if self.stage0_rustfmt is None:
779 return self.program_config('rustfmt')
781 def program_config(self, program, stage0=True):
782 """Return config path for the given program at the given stage
785 >>> rb.config_toml = 'rustc = "rustc"\\n'
786 >>> rb.program_config('rustc')
788 >>> rb.config_toml = ''
789 >>> cargo_path = rb.program_config('cargo', True)
790 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
793 >>> cargo_path = rb.program_config('cargo', False)
794 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
798 config = self.get_toml(program)
800 return os.path.expanduser(config)
801 return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
802 program, self.exe_suffix()))
805 def get_string(line):
806 """Return the value between double quotes
808 >>> RustBuild.get_string(' "devel" ')
810 >>> RustBuild.get_string(" 'devel' ")
812 >>> RustBuild.get_string('devel') is None
814 >>> RustBuild.get_string(' "devel ')
817 start = line.find('"')
819 end = start + 1 + line[start + 1:].find('"')
820 return line[start + 1:end]
821 start = line.find('\'')
823 end = start + 1 + line[start + 1:].find('\'')
824 return line[start + 1:end]
829 """Return a suffix for executables"""
830 if sys.platform == 'win32':
834 def bootstrap_binary(self):
835 """Return the path of the bootstrap binary
838 >>> rb.build_dir = "build"
839 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
840 ... "debug", "bootstrap")
843 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
845 def build_bootstrap(self):
846 """Build bootstrap"""
847 print("Building rustbuild")
848 build_dir = os.path.join(self.build_dir, "bootstrap")
849 if self.clean and os.path.exists(build_dir):
850 shutil.rmtree(build_dir)
851 env = os.environ.copy()
852 # `CARGO_BUILD_TARGET` breaks bootstrap build.
853 # See also: <https://github.com/rust-lang/rust/issues/70208>.
854 if "CARGO_BUILD_TARGET" in env:
855 del env["CARGO_BUILD_TARGET"]
856 env["CARGO_TARGET_DIR"] = build_dir
857 env["RUSTC"] = self.rustc(True)
858 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
859 (os.pathsep + env["LD_LIBRARY_PATH"]) \
860 if "LD_LIBRARY_PATH" in env else ""
861 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
862 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
863 if "DYLD_LIBRARY_PATH" in env else ""
864 env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
865 (os.pathsep + env["LIBRARY_PATH"]) \
866 if "LIBRARY_PATH" in env else ""
868 # preserve existing RUSTFLAGS
869 env.setdefault("RUSTFLAGS", "")
870 build_section = "target.{}".format(self.build)
872 if self.get_toml("crt-static", build_section) == "true":
873 target_features += ["+crt-static"]
874 elif self.get_toml("crt-static", build_section) == "false":
875 target_features += ["-crt-static"]
877 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
878 target_linker = self.get_toml("linker", build_section)
879 if target_linker is not None:
880 env["RUSTFLAGS"] += " -C linker=" + target_linker
881 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
882 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
883 if self.get_toml("deny-warnings", "rust") != "false":
884 env["RUSTFLAGS"] += " -Dwarnings"
886 env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
887 os.pathsep + env["PATH"]
888 if not os.path.isfile(self.cargo()):
889 raise Exception("no cargo executable found at `{}`".format(
891 args = [self.cargo(), "build", "--manifest-path",
892 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
893 for _ in range(0, self.verbose):
894 args.append("--verbose")
895 if self.use_locked_deps:
896 args.append("--locked")
897 if self.use_vendored_sources:
898 args.append("--frozen")
899 run(args, env=env, verbose=self.verbose)
901 def build_triple(self):
902 """Build triple as in LLVM
904 Note that `default_build_triple` is moderately expensive,
905 so use `self.build` where possible.
907 config = self.get_toml('build')
910 return default_build_triple(self.verbose)
912 def check_submodule(self, module, slow_submodules):
913 if not slow_submodules:
914 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
915 cwd=os.path.join(self.rust_root, module),
916 stdout=subprocess.PIPE)
921 def update_submodule(self, module, checked_out, recorded_submodules):
922 module_path = os.path.join(self.rust_root, module)
924 if checked_out is not None:
925 default_encoding = sys.getdefaultencoding()
926 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
927 if recorded_submodules[module] == checked_out:
930 print("Updating submodule", module)
932 run(["git", "submodule", "-q", "sync", module],
933 cwd=self.rust_root, verbose=self.verbose)
935 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
936 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
937 update_args.append("--progress")
938 update_args.append(module)
940 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
942 print("Failed updating submodule. This is probably due to uncommitted local changes.")
943 print('Either stash the changes by running "git stash" within the submodule\'s')
944 print('directory, reset them by running "git reset --hard", or commit them.')
945 print("To reset all submodules' changes run", end=" ")
946 print('"git submodule foreach --recursive git reset --hard".')
949 run(["git", "reset", "-q", "--hard"],
950 cwd=module_path, verbose=self.verbose)
951 run(["git", "clean", "-qdfx"],
952 cwd=module_path, verbose=self.verbose)
954 def update_submodules(self):
955 """Update submodules"""
956 has_git = os.path.exists(os.path.join(self.rust_root, ".git"))
957 # This just arbitrarily checks for cargo, but any workspace member in
958 # a submodule would work.
959 has_submodules = os.path.exists(os.path.join(self.rust_root, "src/tools/cargo/Cargo.toml"))
960 if not has_git and not has_submodules:
961 print("This is not a git repository, and the requisite git submodules were not found.")
962 print("If you downloaded the source from https://github.com/rust-lang/rust/releases,")
963 print("those sources will not work. Instead, consider downloading from the source")
964 print("releases linked at")
965 print("https://forge.rust-lang.org/infra/other-installation-methods.html#source-code")
966 print("or clone the repository at https://github.com/rust-lang/rust/.")
968 if not has_git or self.get_toml('submodules') == "false":
971 default_encoding = sys.getdefaultencoding()
973 # check the existence and version of 'git' command
974 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
975 self.git_version = distutils.version.LooseVersion(git_version_str)
977 slow_submodules = self.get_toml('fast-submodules') == "false"
980 print('Unconditionally updating submodules')
982 print('Updating only changed submodules')
983 default_encoding = sys.getdefaultencoding()
984 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
985 # currently requires everything in a workspace to be "locally present" when starting a
986 # build, and will give a hard error if any Cargo.toml files are missing.
987 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
988 # share a workspace with any tools - maybe it could be excluded from the workspace?
989 # That will still require cloning the submodules the second you check the standard
991 # FIXME: Is there a way to avoid hard-coding the submodules required?
992 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
994 "src/tools/rust-installer",
1001 # If build.vendor is set in config.toml, we must update rust-analyzer also.
1002 # Otherwise, the bootstrap will fail (#96456).
1003 if self.use_vendored_sources:
1004 submodules.append("src/tools/rust-analyzer")
1005 filtered_submodules = []
1006 submodules_names = []
1007 for module in submodules:
1008 check = self.check_submodule(module, slow_submodules)
1009 filtered_submodules.append((module, check))
1010 submodules_names.append(module)
1011 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
1012 cwd=self.rust_root, stdout=subprocess.PIPE)
1013 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
1014 # { filename: hash }
1015 recorded_submodules = {}
1016 for data in recorded:
1017 # [mode, kind, hash, filename]
1019 recorded_submodules[data[3]] = data[2]
1020 for module in filtered_submodules:
1021 self.update_submodule(module[0], module[1], recorded_submodules)
1022 print(" Submodules updated in %.2f seconds" % (time() - start_time))
1024 def set_dist_environment(self, url):
1025 """Set download URL for normal environment"""
1026 if 'RUSTUP_DIST_SERVER' in os.environ:
1027 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1029 self._download_url = url
1031 def check_vendored_status(self):
1032 """Check that vendoring is configured properly"""
1033 vendor_dir = os.path.join(self.rust_root, 'vendor')
1034 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1035 if os.getuid() == 0:
1036 self.use_vendored_sources = True
1037 print('info: looks like you\'re trying to run this command as root')
1038 print(' and so in order to preserve your $HOME this will now')
1039 print(' use vendored sources by default.')
1040 if not os.path.exists(vendor_dir):
1041 print('error: vendoring required, but vendor directory does not exist.')
1042 print(' Run `cargo vendor` without sudo to initialize the '
1043 'vendor directory.')
1044 raise Exception("{} not found".format(vendor_dir))
1046 if self.use_vendored_sources:
1047 config = ("[source.crates-io]\n"
1048 "replace-with = 'vendored-sources'\n"
1049 "registry = 'https://example.com'\n"
1051 "[source.vendored-sources]\n"
1052 "directory = '{}/vendor'\n"
1053 .format(self.rust_root))
1054 if not os.path.exists('.cargo'):
1055 os.makedirs('.cargo')
1056 with output('.cargo/config') as cargo_config:
1057 cargo_config.write(config)
1059 print('info: using vendored source, but .cargo/config is already present.')
1060 print(' Reusing the current configuration file. But you may want to '
1061 'configure vendoring like this:')
1064 if os.path.exists('.cargo'):
1065 shutil.rmtree('.cargo')
1067 def ensure_vendored(self):
1068 """Ensure that the vendored sources are available if needed"""
1069 vendor_dir = os.path.join(self.rust_root, 'vendor')
1070 # Note that this does not handle updating the vendored dependencies if
1071 # the rust git repository is updated. Normal development usually does
1072 # not use vendoring, so hopefully this isn't too much of a problem.
1073 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1077 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1078 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1079 ], verbose=self.verbose, cwd=self.rust_root)
1082 def bootstrap(help_triggered):
1083 """Configure, fetch, build and run the initial bootstrap"""
1085 # If the user is asking for help, let them know that the whole download-and-build
1086 # process has to happen before anything is printed out.
1088 print("info: Downloading and building bootstrap before processing --help")
1089 print(" command. See src/bootstrap/README.md for help with common")
1092 parser = argparse.ArgumentParser(description='Build rust')
1093 parser.add_argument('--config')
1094 parser.add_argument('--build')
1095 parser.add_argument('--clean', action='store_true')
1096 parser.add_argument('-v', '--verbose', action='count', default=0)
1098 args = [a for a in sys.argv if a != '-h' and a != '--help']
1099 args, _ = parser.parse_known_args(args)
1101 # Configure initial bootstrap
1103 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1104 build.verbose = args.verbose
1105 build.clean = args.clean
1107 # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1108 # then `config.toml` in the root directory.
1109 toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1110 using_default_path = toml_path is None
1111 if using_default_path:
1112 toml_path = 'config.toml'
1113 if not os.path.exists(toml_path):
1114 toml_path = os.path.join(build.rust_root, toml_path)
1116 # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1117 # but not if `config.toml` hasn't been created.
1118 if not using_default_path or os.path.exists(toml_path):
1119 with open(toml_path) as config:
1120 build.config_toml = config.read()
1122 profile = build.get_toml('profile')
1123 if profile is not None:
1124 include_file = 'config.{}.toml'.format(profile)
1125 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1126 include_path = os.path.join(include_dir, include_file)
1127 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1128 # specific key, so appending our defaults at the end allows the user to override them
1129 with open(include_path) as included_toml:
1130 build.config_toml += os.linesep + included_toml.read()
1132 config_verbose = build.get_toml('verbose', 'build')
1133 if config_verbose is not None:
1134 build.verbose = max(build.verbose, int(config_verbose))
1136 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1138 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1140 build.check_vendored_status()
1142 build_dir = build.get_toml('build-dir', 'build') or 'build'
1143 build.build_dir = os.path.abspath(build_dir)
1145 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1147 build.checksums_sha256 = data["checksums_sha256"]
1148 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1149 if data.get("rustfmt") is not None:
1150 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1152 build.set_dist_environment(data["dist_server"])
1154 build.build = args.build or build.build_triple()
1156 # Acquire the lock before doing any build actions
1157 # The lock is released when `lock` is dropped
1158 if not os.path.exists(build.build_dir):
1159 os.makedirs(build.build_dir)
1160 lock = acquire_lock(build.build_dir)
1161 build.update_submodules()
1163 # Fetch/build the bootstrap
1164 build.download_toolchain()
1165 # Download the master compiler if `download-rustc` is set
1166 build.maybe_download_ci_toolchain()
1168 build.ensure_vendored()
1169 build.build_bootstrap()
1173 args = [build.bootstrap_binary()]
1174 args.extend(sys.argv[1:])
1175 env = os.environ.copy()
1176 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1177 env["BOOTSTRAP_PYTHON"] = sys.executable
1178 env["RUSTC_BOOTSTRAP"] = '1'
1179 if build.rustc_commit is not None:
1180 env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
1181 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1185 """Entry point for the bootstrap process"""
1188 # x.py help <cmd> ...
1189 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1190 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1193 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1195 bootstrap(help_triggered)
1196 if not help_triggered:
1197 print("Build completed successfully in {}".format(
1198 format_build_time(time() - start_time)))
1199 except (SystemExit, KeyboardInterrupt) as error:
1200 if hasattr(error, 'code') and isinstance(error.code, int):
1201 exit_code = error.code
1205 if not help_triggered:
1206 print("Build completed unsuccessfully in {}".format(
1207 format_build_time(time() - start_time)))
1211 if __name__ == '__main__':