1 from __future__ import absolute_import, division, print_function
5 import distutils.version
16 from time import time, sleep
18 # Acquire a lock on the build directory to make sure that
19 # we don't cause a race condition while building
20 # Lock is created in `build_dir/lock.db`
21 def acquire_lock(build_dir):
25 path = os.path.join(build_dir, "lock.db")
27 con = sqlite3.Connection(path, timeout=0)
29 curs.execute("BEGIN EXCLUSIVE")
30 # The lock is released when the cursor is dropped
32 # If the database is busy then lock has already been acquired
33 # so we wait for the lock.
34 # We retry every quarter second so that execution is passed back to python
35 # so that it can handle signals
36 except sqlite3.OperationalError:
39 print("Waiting for lock on build directory")
40 con = sqlite3.Connection(path, timeout=0.25)
44 curs.execute("BEGIN EXCLUSIVE")
46 except sqlite3.OperationalError:
51 print("warning: sqlite3 not available in python, skipping build directory lock")
52 print("please file an issue on rust-lang/rust")
53 print("this is not a problem for non-concurrent x.py invocations")
58 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
59 temp_path = temp_file.name
60 with tarfile.open(temp_path, "w:xz"):
63 except tarfile.CompressionError:
66 def get(base, url, path, checksums, verbose=False, do_verify=True):
67 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
68 temp_path = temp_file.name
72 if url not in checksums:
73 raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
74 "Pre-built artifacts might not available for this "
75 "target at this time, see https://doc.rust-lang.org/nightly"
76 "/rustc/platform-support.html for more information.")
78 sha256 = checksums[url]
79 if os.path.exists(path):
80 if verify(path, sha256, False):
82 print("using already-download file", path)
86 print("ignoring already-download file",
87 path, "due to failed verification")
89 download(temp_path, "{}/{}".format(base, url), True, verbose)
90 if do_verify and not verify(temp_path, sha256, verbose):
91 raise RuntimeError("failed verification")
93 print("moving {} to {}".format(temp_path, path))
94 shutil.move(temp_path, path)
96 if os.path.isfile(temp_path):
98 print("removing", temp_path)
102 def download(path, url, probably_big, verbose):
103 for _ in range(0, 4):
105 _download(path, url, probably_big, verbose, True)
108 print("\nspurious failure, trying again")
109 _download(path, url, probably_big, verbose, False)
112 def _download(path, url, probably_big, verbose, exception):
113 # Try to use curl (potentially available on win32
114 # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/)
115 # If an error occurs:
116 # - If we are on win32 fallback to powershell
117 # - Otherwise raise the error if appropriate
118 if probably_big or verbose:
119 print("downloading {}".format(url))
121 platform_is_win32 = sys.platform == 'win32'
123 if probably_big or verbose:
127 # If curl is not present on Win32, we shoud not sys.exit
128 # but raise `CalledProcessError` or `OSError` instead
129 require(["curl", "--version"], exception=platform_is_win32)
131 "-L", # Follow redirect.
132 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
133 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
134 "--retry", "3", "-Sf", "-o", path, url],
136 exception=True, # Will raise RuntimeError on failure
138 except (subprocess.CalledProcessError, OSError, RuntimeError):
139 # see http://serverfault.com/questions/301128/how-to-download
140 if platform_is_win32:
141 run(["PowerShell.exe", "/nologo", "-Command",
142 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
143 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
146 # Check if the RuntimeError raised by run(curl) should be silenced
147 elif verbose or exception:
151 def verify(path, expected, verbose):
152 """Check if the sha256 sum of the given path is valid"""
154 print("verifying", path)
155 with open(path, "rb") as source:
156 found = hashlib.sha256(source.read()).hexdigest()
157 verified = found == expected
159 print("invalid checksum:\n"
161 " expected: {}".format(found, expected))
165 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
166 """Unpack the given tarball file"""
167 print("extracting", tarball)
168 fname = os.path.basename(tarball).replace(tarball_suffix, "")
169 with contextlib.closing(tarfile.open(tarball)) as tar:
170 for member in tar.getnames():
171 if "/" not in member:
173 name = member.replace(fname + "/", "", 1)
174 if match is not None and not name.startswith(match):
176 name = name[len(match) + 1:]
178 dst_path = os.path.join(dst, name)
180 print(" extracting", member)
181 tar.extract(member, dst)
182 src_path = os.path.join(dst, member)
183 if os.path.isdir(src_path) and os.path.exists(dst_path):
185 shutil.move(src_path, dst_path)
186 shutil.rmtree(os.path.join(dst, fname))
189 def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
190 """Run a child program in a new process"""
192 print("running: " + ' '.join(args))
194 # Use Popen here instead of call() as it apparently allows powershell on
195 # Windows to not lock up waiting for input presumably.
196 ret = subprocess.Popen(args, **kwargs)
199 err = "failed to run: " + ' '.join(args)
200 if verbose or exception:
201 raise RuntimeError(err)
202 # For most failures, we definitely do want to print this error, or the user will have no
203 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
204 # have already printed an error above, so there's no need to print the exact command we're
212 def require(cmd, exit=True, exception=False):
213 '''Run a command, returning its output.
215 If `exception` is `True`, raise the error
216 Otherwise If `exit` is `True`, exit the process
219 return subprocess.check_output(cmd).strip()
220 except (subprocess.CalledProcessError, OSError) as exc:
224 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
225 print("Please make sure it's installed and in the path.")
231 def format_build_time(duration):
232 """Return a nicer format for build time
234 >>> format_build_time('300')
237 return str(datetime.timedelta(seconds=int(duration)))
240 def default_build_triple(verbose):
241 """Build triple as in LLVM"""
242 # If the user already has a host build triple with an existing `rustc`
243 # install, use their preference. This fixes most issues with Windows builds
244 # being detected as GNU instead of MSVC.
245 default_encoding = sys.getdefaultencoding()
247 version = subprocess.check_output(["rustc", "--version", "--verbose"],
248 stderr=subprocess.DEVNULL)
249 version = version.decode(default_encoding)
250 host = next(x for x in version.split('\n') if x.startswith("host: "))
251 triple = host.split("host: ")[1]
253 print("detected default triple {} from pre-installed rustc".format(triple))
255 except Exception as e:
257 print("pre-installed rustc not detected: {}".format(e))
258 print("falling back to auto-detect")
260 required = sys.platform != 'win32'
261 ostype = require(["uname", "-s"], exit=required)
262 cputype = require(['uname', '-m'], exit=required)
264 # If we do not have `uname`, assume Windows.
265 if ostype is None or cputype is None:
266 return 'x86_64-pc-windows-msvc'
268 ostype = ostype.decode(default_encoding)
269 cputype = cputype.decode(default_encoding)
271 # The goal here is to come up with the same triple as LLVM would,
272 # at least for the subset of platforms we're willing to target.
274 'Darwin': 'apple-darwin',
275 'DragonFly': 'unknown-dragonfly',
276 'FreeBSD': 'unknown-freebsd',
277 'Haiku': 'unknown-haiku',
278 'NetBSD': 'unknown-netbsd',
279 'OpenBSD': 'unknown-openbsd'
282 # Consider the direct transformation first and then the special cases
283 if ostype in ostype_mapper:
284 ostype = ostype_mapper[ostype]
285 elif ostype == 'Linux':
286 os_from_sp = subprocess.check_output(
287 ['uname', '-o']).strip().decode(default_encoding)
288 if os_from_sp == 'Android':
289 ostype = 'linux-android'
291 ostype = 'unknown-linux-gnu'
292 elif ostype == 'SunOS':
293 ostype = 'pc-solaris'
294 # On Solaris, uname -m will return a machine classification instead
295 # of a cpu type, so uname -p is recommended instead. However, the
296 # output from that option is too generic for our purposes (it will
297 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
298 # must be used instead.
299 cputype = require(['isainfo', '-k']).decode(default_encoding)
300 # sparc cpus have sun as a target vendor
301 if 'sparc' in cputype:
302 ostype = 'sun-solaris'
303 elif ostype.startswith('MINGW'):
304 # msys' `uname` does not print gcc configuration, but prints msys
305 # configuration. so we cannot believe `uname -m`:
306 # msys1 is always i686 and msys2 is always x86_64.
307 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
309 ostype = 'pc-windows-gnu'
311 if os.environ.get('MSYSTEM') == 'MINGW64':
313 elif ostype.startswith('MSYS'):
314 ostype = 'pc-windows-gnu'
315 elif ostype.startswith('CYGWIN_NT'):
317 if ostype.endswith('WOW64'):
319 ostype = 'pc-windows-gnu'
320 elif sys.platform == 'win32':
321 # Some Windows platforms might have a `uname` command that returns a
322 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
323 # these cases, fall back to using sys.platform.
324 return 'x86_64-pc-windows-msvc'
326 err = "unknown OS type: {}".format(ostype)
329 if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd':
330 cputype = subprocess.check_output(
331 ['uname', '-p']).strip().decode(default_encoding)
334 'aarch64': 'aarch64',
342 'powerpc': 'powerpc',
343 'powerpc64': 'powerpc64',
344 'powerpc64le': 'powerpc64le',
346 'ppc64': 'powerpc64',
347 'ppc64le': 'powerpc64le',
348 'riscv64': 'riscv64gc',
356 # Consider the direct transformation first and then the special cases
357 if cputype in cputype_mapper:
358 cputype = cputype_mapper[cputype]
359 elif cputype in {'xscale', 'arm'}:
361 if ostype == 'linux-android':
362 ostype = 'linux-androideabi'
363 elif ostype == 'unknown-freebsd':
364 cputype = subprocess.check_output(
365 ['uname', '-p']).strip().decode(default_encoding)
366 ostype = 'unknown-freebsd'
367 elif cputype == 'armv6l':
369 if ostype == 'linux-android':
370 ostype = 'linux-androideabi'
373 elif cputype in {'armv7l', 'armv8l'}:
375 if ostype == 'linux-android':
376 ostype = 'linux-androideabi'
379 elif cputype == 'mips':
380 if sys.byteorder == 'big':
382 elif sys.byteorder == 'little':
385 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
386 elif cputype == 'mips64':
387 if sys.byteorder == 'big':
389 elif sys.byteorder == 'little':
392 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
393 # only the n64 ABI is supported, indicate it
395 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
398 err = "unknown cpu type: {}".format(cputype)
401 return "{}-{}".format(cputype, ostype)
404 @contextlib.contextmanager
405 def output(filepath):
406 tmp = filepath + '.tmp'
407 with open(tmp, 'w') as f:
410 if os.path.exists(filepath):
411 os.remove(filepath) # PermissionError/OSError on Win32 if in use
413 shutil.copy2(tmp, filepath)
416 os.rename(tmp, filepath)
419 class Stage0Toolchain:
420 def __init__(self, stage0_payload):
421 self.date = stage0_payload["date"]
422 self.version = stage0_payload["version"]
425 return self.version + "-" + self.date
428 class RustBuild(object):
429 """Provide all the methods required to build Rust"""
431 self.checksums_sha256 = {}
432 self.stage0_compiler = None
433 self.stage0_rustfmt = None
434 self._download_url = ''
438 self.config_toml = ''
440 self.use_locked_deps = ''
441 self.use_vendored_sources = ''
443 self.git_version = None
444 self.nix_deps_dir = None
446 def download_toolchain(self):
447 """Fetch the build system for Rust, written in Rust
449 This method will build a cache directory, then it will fetch the
450 tarball which has the stage0 compiler used to then bootstrap the Rust
453 Each downloaded tarball is extracted, after that, the script
454 will move all the content to the right place.
456 rustc_channel = self.stage0_compiler.version
457 bin_root = self.bin_root()
459 key = self.stage0_compiler.date
460 if self.rustc().startswith(bin_root) and \
461 (not os.path.exists(self.rustc()) or
462 self.program_out_of_date(self.rustc_stamp(), key)):
463 if os.path.exists(bin_root):
464 shutil.rmtree(bin_root)
465 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
466 filename = "rust-std-{}-{}{}".format(
467 rustc_channel, self.build, tarball_suffix)
468 pattern = "rust-std-{}".format(self.build)
469 self._download_component_helper(filename, pattern, tarball_suffix)
470 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
472 self._download_component_helper(filename, "rustc", tarball_suffix)
473 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
475 self._download_component_helper(filename, "cargo", tarball_suffix)
476 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
478 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
479 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
480 lib_dir = "{}/lib".format(bin_root)
481 for lib in os.listdir(lib_dir):
482 if lib.endswith(".so"):
483 self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
484 with output(self.rustc_stamp()) as rust_stamp:
485 rust_stamp.write(key)
487 if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
488 not os.path.exists(self.rustfmt())
489 or self.program_out_of_date(
490 self.rustfmt_stamp(),
491 "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel()
494 if self.stage0_rustfmt is not None:
495 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
496 filename = "rustfmt-{}-{}{}".format(
497 self.stage0_rustfmt.version, self.build, tarball_suffix,
499 self._download_component_helper(
500 filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date
502 self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
503 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
504 with output(self.rustfmt_stamp()) as rustfmt_stamp:
505 rustfmt_stamp.write(self.stage0_rustfmt.channel())
507 def _download_component_helper(
508 self, filename, pattern, tarball_suffix, key=None
511 key = self.stage0_compiler.date
512 cache_dst = os.path.join(self.build_dir, "cache")
513 rustc_cache = os.path.join(cache_dst, key)
514 if not os.path.exists(rustc_cache):
515 os.makedirs(rustc_cache)
517 base = self._download_url
518 url = "dist/{}".format(key)
519 tarball = os.path.join(rustc_cache, filename)
520 if not os.path.exists(tarball):
523 "{}/{}".format(url, filename),
525 self.checksums_sha256,
526 verbose=self.verbose,
529 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
531 def fix_bin_or_dylib(self, fname):
532 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
533 or the RPATH section, to fix the dynamic library search path
535 This method is only required on NixOS and uses the PatchELF utility to
536 change the interpreter/RPATH of ELF executables.
538 Please see https://nixos.org/patchelf.html for more information
540 default_encoding = sys.getdefaultencoding()
542 ostype = subprocess.check_output(
543 ['uname', '-s']).strip().decode(default_encoding)
544 except subprocess.CalledProcessError:
546 except OSError as reason:
547 if getattr(reason, 'winerror', None) is not None:
551 if ostype != "Linux":
554 # If the user has asked binaries to be patched for Nix, then
555 # don't check for NixOS or `/lib`, just continue to the patching.
556 if self.get_toml('patch-binaries-for-nix', 'build') != 'true':
557 # Use `/etc/os-release` instead of `/etc/NIXOS`.
558 # The latter one does not exist on NixOS when using tmpfs as root.
560 with open("/etc/os-release", "r") as f:
561 if not any(l.strip() in ["ID=nixos", "ID='nixos'", 'ID="nixos"'] for l in f):
563 except FileNotFoundError:
565 if os.path.exists("/lib"):
568 # At this point we're pretty sure the user is running NixOS or
570 nix_os_msg = "info: you seem to be using Nix. Attempting to patch"
571 print(nix_os_msg, fname)
573 # Only build `.nix-deps` once.
574 nix_deps_dir = self.nix_deps_dir
576 # Run `nix-build` to "build" each dependency (which will likely reuse
577 # the existing `/nix/store` copy, or at most download a pre-built copy).
579 # Importantly, we create a gc-root called `.nix-deps` in the `build/`
580 # directory, but still reference the actual `/nix/store` path in the rpath
581 # as it makes it significantly more robust against changes to the location of
582 # the `.nix-deps` location.
584 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
585 # zlib: Needed as a system dependency of `libLLVM-*.so`.
586 # patchelf: Needed for patching ELF binaries (see doc comment above).
587 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
589 with (import <nixpkgs> {});
591 name = "rust-stage0-dependencies";
600 subprocess.check_output([
601 "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
603 except subprocess.CalledProcessError as reason:
604 print("warning: failed to call nix-build:", reason)
606 self.nix_deps_dir = nix_deps_dir
608 patchelf = "{}/bin/patchelf".format(nix_deps_dir)
610 # Relative default, all binary and dynamic libraries we ship
611 # appear to have this (even when `../lib` is redundant).
613 os.path.join(os.path.realpath(nix_deps_dir), "lib")
615 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
616 if not fname.endswith(".so"):
617 # Finally, set the corret .interp for binaries
618 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
619 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
622 subprocess.check_output([patchelf] + patchelf_args + [fname])
623 except subprocess.CalledProcessError as reason:
624 print("warning: failed to call patchelf:", reason)
627 def rustc_stamp(self):
628 """Return the path for .rustc-stamp at the given stage
631 >>> rb.build_dir = "build"
632 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
635 return os.path.join(self.bin_root(), '.rustc-stamp')
637 def rustfmt_stamp(self):
638 """Return the path for .rustfmt-stamp
641 >>> rb.build_dir = "build"
642 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
645 return os.path.join(self.bin_root(), '.rustfmt-stamp')
647 def program_out_of_date(self, stamp_path, key):
648 """Check if the given program stamp is out of date"""
649 if not os.path.exists(stamp_path) or self.clean:
651 with open(stamp_path, 'r') as stamp:
652 return key != stamp.read()
655 """Return the binary root directory for the given stage
658 >>> rb.build_dir = "build"
659 >>> rb.bin_root() == os.path.join("build", "stage0")
662 When the 'build' property is given should be a nested directory:
664 >>> rb.build = "devel"
665 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
669 return os.path.join(self.build_dir, self.build, subdir)
671 def get_toml(self, key, section=None):
672 """Returns the value of the given key in config.toml, otherwise returns None
675 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
676 >>> rb.get_toml("key2")
679 If the key does not exist, the result is None:
681 >>> rb.get_toml("key3") is None
684 Optionally also matches the section the key appears in
686 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
687 >>> rb.get_toml('key', 'a')
689 >>> rb.get_toml('key', 'b')
691 >>> rb.get_toml('key', 'c') is None
694 >>> rb.config_toml = 'key1 = true'
695 >>> rb.get_toml("key1")
700 for line in self.config_toml.splitlines():
701 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
702 if section_match is not None:
703 cur_section = section_match.group(1)
705 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
706 if match is not None:
707 value = match.group(1)
708 if section is None or section == cur_section:
709 return self.get_string(value) or value.strip()
713 """Return config path for cargo"""
714 return self.program_config('cargo')
717 """Return config path for rustc"""
718 return self.program_config('rustc')
721 """Return config path for rustfmt"""
722 if self.stage0_rustfmt is None:
724 return self.program_config('rustfmt')
726 def program_config(self, program):
727 """Return config path for the given program at the given stage
730 >>> rb.config_toml = 'rustc = "rustc"\\n'
731 >>> rb.program_config('rustc')
733 >>> rb.config_toml = ''
734 >>> cargo_path = rb.program_config('cargo')
735 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
739 config = self.get_toml(program)
741 return os.path.expanduser(config)
742 return os.path.join(self.bin_root(), "bin", "{}{}".format(
743 program, self.exe_suffix()))
746 def get_string(line):
747 """Return the value between double quotes
749 >>> RustBuild.get_string(' "devel" ')
751 >>> RustBuild.get_string(" 'devel' ")
753 >>> RustBuild.get_string('devel') is None
755 >>> RustBuild.get_string(' "devel ')
758 start = line.find('"')
760 end = start + 1 + line[start + 1:].find('"')
761 return line[start + 1:end]
762 start = line.find('\'')
764 end = start + 1 + line[start + 1:].find('\'')
765 return line[start + 1:end]
770 """Return a suffix for executables"""
771 if sys.platform == 'win32':
775 def bootstrap_binary(self):
776 """Return the path of the bootstrap binary
779 >>> rb.build_dir = "build"
780 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
781 ... "debug", "bootstrap")
784 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
786 def build_bootstrap(self):
787 """Build bootstrap"""
788 print("Building rustbuild")
789 build_dir = os.path.join(self.build_dir, "bootstrap")
790 if self.clean and os.path.exists(build_dir):
791 shutil.rmtree(build_dir)
792 env = os.environ.copy()
793 # `CARGO_BUILD_TARGET` breaks bootstrap build.
794 # See also: <https://github.com/rust-lang/rust/issues/70208>.
795 if "CARGO_BUILD_TARGET" in env:
796 del env["CARGO_BUILD_TARGET"]
797 env["CARGO_TARGET_DIR"] = build_dir
798 env["RUSTC"] = self.rustc()
799 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
800 (os.pathsep + env["LD_LIBRARY_PATH"]) \
801 if "LD_LIBRARY_PATH" in env else ""
802 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
803 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
804 if "DYLD_LIBRARY_PATH" in env else ""
805 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
806 (os.pathsep + env["LIBRARY_PATH"]) \
807 if "LIBRARY_PATH" in env else ""
809 # preserve existing RUSTFLAGS
810 env.setdefault("RUSTFLAGS", "")
811 build_section = "target.{}".format(self.build)
813 if self.get_toml("crt-static", build_section) == "true":
814 target_features += ["+crt-static"]
815 elif self.get_toml("crt-static", build_section) == "false":
816 target_features += ["-crt-static"]
818 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
819 target_linker = self.get_toml("linker", build_section)
820 if target_linker is not None:
821 env["RUSTFLAGS"] += " -C linker=" + target_linker
822 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
823 env["RUSTFLAGS"] += " -Wsemicolon_in_expressions_from_macros"
824 if self.get_toml("deny-warnings", "rust") != "false":
825 env["RUSTFLAGS"] += " -Dwarnings"
827 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
828 os.pathsep + env["PATH"]
829 if not os.path.isfile(self.cargo()):
830 raise Exception("no cargo executable found at `{}`".format(
832 args = [self.cargo(), "build", "--manifest-path",
833 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
834 for _ in range(0, self.verbose):
835 args.append("--verbose")
836 if self.use_locked_deps:
837 args.append("--locked")
838 if self.use_vendored_sources:
839 args.append("--frozen")
840 if self.get_toml("metrics", "build"):
841 args.append("--features")
842 args.append("build-metrics")
843 run(args, env=env, verbose=self.verbose)
845 def build_triple(self):
846 """Build triple as in LLVM
848 Note that `default_build_triple` is moderately expensive,
849 so use `self.build` where possible.
851 config = self.get_toml('build')
854 return default_build_triple(self.verbose)
856 def check_submodule(self, module):
857 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
858 cwd=os.path.join(self.rust_root, module),
859 stdout=subprocess.PIPE)
862 def update_submodule(self, module, checked_out, recorded_submodules):
863 module_path = os.path.join(self.rust_root, module)
865 default_encoding = sys.getdefaultencoding()
866 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
867 if recorded_submodules[module] == checked_out:
870 print("Updating submodule", module)
872 run(["git", "submodule", "-q", "sync", module],
873 cwd=self.rust_root, verbose=self.verbose)
875 update_args = ["git", "submodule", "update", "--init", "--recursive", "--depth=1"]
876 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
877 update_args.append("--progress")
878 update_args.append(module)
880 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
882 print("Failed updating submodule. This is probably due to uncommitted local changes.")
883 print('Either stash the changes by running "git stash" within the submodule\'s')
884 print('directory, reset them by running "git reset --hard", or commit them.')
885 print("To reset all submodules' changes run", end=" ")
886 print('"git submodule foreach --recursive git reset --hard".')
889 run(["git", "reset", "-q", "--hard"],
890 cwd=module_path, verbose=self.verbose)
891 run(["git", "clean", "-qdfx"],
892 cwd=module_path, verbose=self.verbose)
894 def update_submodules(self):
895 """Update submodules"""
896 has_git = os.path.exists(os.path.join(self.rust_root, ".git"))
897 # This just arbitrarily checks for cargo, but any workspace member in
898 # a submodule would work.
899 has_submodules = os.path.exists(os.path.join(self.rust_root, "src/tools/cargo/Cargo.toml"))
900 if not has_git and not has_submodules:
901 print("This is not a git repository, and the requisite git submodules were not found.")
902 print("If you downloaded the source from https://github.com/rust-lang/rust/releases,")
903 print("those sources will not work. Instead, consider downloading from the source")
904 print("releases linked at")
905 print("https://forge.rust-lang.org/infra/other-installation-methods.html#source-code")
906 print("or clone the repository at https://github.com/rust-lang/rust/.")
908 if not has_git or self.get_toml('submodules') == "false":
911 default_encoding = sys.getdefaultencoding()
913 # check the existence and version of 'git' command
914 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
915 self.git_version = distutils.version.LooseVersion(git_version_str)
918 print('Updating only changed submodules')
919 default_encoding = sys.getdefaultencoding()
920 # Only update submodules that are needed to build bootstrap. These are needed because Cargo
921 # currently requires everything in a workspace to be "locally present" when starting a
922 # build, and will give a hard error if any Cargo.toml files are missing.
923 # FIXME: Is there a way to avoid cloning these eagerly? Bootstrap itself doesn't need to
924 # share a workspace with any tools - maybe it could be excluded from the workspace?
925 # That will still require cloning the submodules the second you check the standard
927 # FIXME: Is there a way to avoid hard-coding the submodules required?
928 # WARNING: keep this in sync with the submodules hard-coded in bootstrap/lib.rs
930 "src/tools/rust-installer",
937 # If build.vendor is set in config.toml, we must update rust-analyzer also.
938 # Otherwise, the bootstrap will fail (#96456).
939 if self.use_vendored_sources:
940 submodules.append("src/tools/rust-analyzer")
941 filtered_submodules = []
942 submodules_names = []
943 for module in submodules:
944 check = self.check_submodule(module)
945 filtered_submodules.append((module, check))
946 submodules_names.append(module)
947 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
948 cwd=self.rust_root, stdout=subprocess.PIPE)
949 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
951 recorded_submodules = {}
952 for data in recorded:
953 # [mode, kind, hash, filename]
955 recorded_submodules[data[3]] = data[2]
956 for module in filtered_submodules:
957 self.update_submodule(module[0], module[1], recorded_submodules)
958 print(" Submodules updated in %.2f seconds" % (time() - start_time))
960 def set_dist_environment(self, url):
961 """Set download URL for normal environment"""
962 if 'RUSTUP_DIST_SERVER' in os.environ:
963 self._download_url = os.environ['RUSTUP_DIST_SERVER']
965 self._download_url = url
967 def check_vendored_status(self):
968 """Check that vendoring is configured properly"""
969 vendor_dir = os.path.join(self.rust_root, 'vendor')
970 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
972 self.use_vendored_sources = True
973 print('info: looks like you\'re trying to run this command as root')
974 print(' and so in order to preserve your $HOME this will now')
975 print(' use vendored sources by default.')
976 if not os.path.exists(vendor_dir):
977 print('error: vendoring required, but vendor directory does not exist.')
978 print(' Run `cargo vendor` without sudo to initialize the '
980 raise Exception("{} not found".format(vendor_dir))
982 if self.use_vendored_sources:
983 config = ("[source.crates-io]\n"
984 "replace-with = 'vendored-sources'\n"
985 "registry = 'https://example.com'\n"
987 "[source.vendored-sources]\n"
988 "directory = '{}/vendor'\n"
989 .format(self.rust_root))
990 if not os.path.exists('.cargo'):
991 os.makedirs('.cargo')
992 with output('.cargo/config') as cargo_config:
993 cargo_config.write(config)
995 print('info: using vendored source, but .cargo/config is already present.')
996 print(' Reusing the current configuration file. But you may want to '
997 'configure vendoring like this:')
1000 if os.path.exists('.cargo'):
1001 shutil.rmtree('.cargo')
1003 def ensure_vendored(self):
1004 """Ensure that the vendored sources are available if needed"""
1005 vendor_dir = os.path.join(self.rust_root, 'vendor')
1006 # Note that this does not handle updating the vendored dependencies if
1007 # the rust git repository is updated. Normal development usually does
1008 # not use vendoring, so hopefully this isn't too much of a problem.
1009 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1013 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1014 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1015 ], verbose=self.verbose, cwd=self.rust_root)
1018 def bootstrap(help_triggered):
1019 """Configure, fetch, build and run the initial bootstrap"""
1021 # If the user is asking for help, let them know that the whole download-and-build
1022 # process has to happen before anything is printed out.
1024 print("info: Downloading and building bootstrap before processing --help")
1025 print(" command. See src/bootstrap/README.md for help with common")
1028 parser = argparse.ArgumentParser(description='Build rust')
1029 parser.add_argument('--config')
1030 parser.add_argument('--build')
1031 parser.add_argument('--clean', action='store_true')
1032 parser.add_argument('-v', '--verbose', action='count', default=0)
1034 args = [a for a in sys.argv if a != '-h' and a != '--help']
1035 args, _ = parser.parse_known_args(args)
1037 # Configure initial bootstrap
1039 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1040 build.verbose = args.verbose
1041 build.clean = args.clean
1043 # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1044 # then `config.toml` in the root directory.
1045 toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1046 using_default_path = toml_path is None
1047 if using_default_path:
1048 toml_path = 'config.toml'
1049 if not os.path.exists(toml_path):
1050 toml_path = os.path.join(build.rust_root, toml_path)
1052 # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1053 # but not if `config.toml` hasn't been created.
1054 if not using_default_path or os.path.exists(toml_path):
1055 with open(toml_path) as config:
1056 build.config_toml = config.read()
1058 profile = build.get_toml('profile')
1059 if profile is not None:
1060 include_file = 'config.{}.toml'.format(profile)
1061 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1062 include_path = os.path.join(include_dir, include_file)
1063 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1064 # specific key, so appending our defaults at the end allows the user to override them
1065 with open(include_path) as included_toml:
1066 build.config_toml += os.linesep + included_toml.read()
1068 config_verbose = build.get_toml('verbose', 'build')
1069 if config_verbose is not None:
1070 build.verbose = max(build.verbose, int(config_verbose))
1072 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1074 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1076 build.check_vendored_status()
1078 build_dir = build.get_toml('build-dir', 'build') or 'build'
1079 build.build_dir = os.path.abspath(build_dir)
1081 with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
1083 build.checksums_sha256 = data["checksums_sha256"]
1084 build.stage0_compiler = Stage0Toolchain(data["compiler"])
1085 if data.get("rustfmt") is not None:
1086 build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"])
1088 build.set_dist_environment(data["dist_server"])
1090 build.build = args.build or build.build_triple()
1092 # Acquire the lock before doing any build actions
1093 # The lock is released when `lock` is dropped
1094 if not os.path.exists(build.build_dir):
1095 os.makedirs(build.build_dir)
1096 lock = acquire_lock(build.build_dir)
1097 build.update_submodules()
1099 # Fetch/build the bootstrap
1100 build.download_toolchain()
1102 build.ensure_vendored()
1103 build.build_bootstrap()
1107 args = [build.bootstrap_binary()]
1108 args.extend(sys.argv[1:])
1109 env = os.environ.copy()
1110 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1111 env["BOOTSTRAP_PYTHON"] = sys.executable
1112 run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1116 """Entry point for the bootstrap process"""
1119 # x.py help <cmd> ...
1120 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1121 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1124 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1126 bootstrap(help_triggered)
1127 if not help_triggered:
1128 print("Build completed successfully in {}".format(
1129 format_build_time(time() - start_time)))
1130 except (SystemExit, KeyboardInterrupt) as error:
1131 if hasattr(error, 'code') and isinstance(error.code, int):
1132 exit_code = error.code
1136 if not help_triggered:
1137 print("Build completed unsuccessfully in {}".format(
1138 format_build_time(time() - start_time)))
1142 if __name__ == '__main__':