1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"],
198 stderr=subprocess.DEVNULL)
199 version = version.decode(default_encoding)
200 host = next(x for x in version.split('\n') if x.startswith("host: "))
201 triple = host.split("host: ")[1]
203 print("detected default triple {}".format(triple))
205 except Exception as e:
207 print("rustup not detected: {}".format(e))
208 print("falling back to auto-detect")
210 required = sys.platform != 'win32'
211 ostype = require(["uname", "-s"], exit=required)
212 cputype = require(['uname', '-m'], exit=required)
214 # If we do not have `uname`, assume Windows.
215 if ostype is None or cputype is None:
216 return 'x86_64-pc-windows-msvc'
218 ostype = ostype.decode(default_encoding)
219 cputype = cputype.decode(default_encoding)
221 # The goal here is to come up with the same triple as LLVM would,
222 # at least for the subset of platforms we're willing to target.
224 'Darwin': 'apple-darwin',
225 'DragonFly': 'unknown-dragonfly',
226 'FreeBSD': 'unknown-freebsd',
227 'Haiku': 'unknown-haiku',
228 'NetBSD': 'unknown-netbsd',
229 'OpenBSD': 'unknown-openbsd'
232 # Consider the direct transformation first and then the special cases
233 if ostype in ostype_mapper:
234 ostype = ostype_mapper[ostype]
235 elif ostype == 'Linux':
236 os_from_sp = subprocess.check_output(
237 ['uname', '-o']).strip().decode(default_encoding)
238 if os_from_sp == 'Android':
239 ostype = 'linux-android'
241 ostype = 'unknown-linux-gnu'
242 elif ostype == 'SunOS':
243 ostype = 'sun-solaris'
244 # On Solaris, uname -m will return a machine classification instead
245 # of a cpu type, so uname -p is recommended instead. However, the
246 # output from that option is too generic for our purposes (it will
247 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
248 # must be used instead.
249 cputype = require(['isainfo', '-k']).decode(default_encoding)
250 elif ostype.startswith('MINGW'):
251 # msys' `uname` does not print gcc configuration, but prints msys
252 # configuration. so we cannot believe `uname -m`:
253 # msys1 is always i686 and msys2 is always x86_64.
254 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
256 ostype = 'pc-windows-gnu'
258 if os.environ.get('MSYSTEM') == 'MINGW64':
260 elif ostype.startswith('MSYS'):
261 ostype = 'pc-windows-gnu'
262 elif ostype.startswith('CYGWIN_NT'):
264 if ostype.endswith('WOW64'):
266 ostype = 'pc-windows-gnu'
267 elif sys.platform == 'win32':
268 # Some Windows platforms might have a `uname` command that returns a
269 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
270 # these cases, fall back to using sys.platform.
271 return 'x86_64-pc-windows-msvc'
273 err = "unknown OS type: {}".format(ostype)
276 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
277 cputype = subprocess.check_output(
278 ['uname', '-p']).strip().decode(default_encoding)
281 'aarch64': 'aarch64',
288 'powerpc': 'powerpc',
289 'powerpc64': 'powerpc64',
290 'powerpc64le': 'powerpc64le',
292 'ppc64': 'powerpc64',
293 'ppc64le': 'powerpc64le',
301 # Consider the direct transformation first and then the special cases
302 if cputype in cputype_mapper:
303 cputype = cputype_mapper[cputype]
304 elif cputype in {'xscale', 'arm'}:
306 if ostype == 'linux-android':
307 ostype = 'linux-androideabi'
308 elif ostype == 'unknown-freebsd':
309 cputype = subprocess.check_output(
310 ['uname', '-p']).strip().decode(default_encoding)
311 ostype = 'unknown-freebsd'
312 elif cputype == 'armv6l':
314 if ostype == 'linux-android':
315 ostype = 'linux-androideabi'
318 elif cputype in {'armv7l', 'armv8l'}:
320 if ostype == 'linux-android':
321 ostype = 'linux-androideabi'
324 elif cputype == 'mips':
325 if sys.byteorder == 'big':
327 elif sys.byteorder == 'little':
330 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
331 elif cputype == 'mips64':
332 if sys.byteorder == 'big':
334 elif sys.byteorder == 'little':
337 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
338 # only the n64 ABI is supported, indicate it
340 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
343 err = "unknown cpu type: {}".format(cputype)
346 return "{}-{}".format(cputype, ostype)
349 @contextlib.contextmanager
350 def output(filepath):
351 tmp = filepath + '.tmp'
352 with open(tmp, 'w') as f:
355 if os.path.exists(filepath):
356 os.remove(filepath) # PermissionError/OSError on Win32 if in use
358 shutil.copy2(tmp, filepath)
361 os.rename(tmp, filepath)
364 class RustBuild(object):
365 """Provide all the methods required to build Rust"""
368 self._download_url = ''
369 self.rustc_channel = ''
370 self.rustfmt_channel = ''
374 self.config_toml = ''
376 self.use_locked_deps = ''
377 self.use_vendored_sources = ''
379 self.git_version = None
380 self.nix_deps_dir = None
381 self.rustc_commit = None
383 def download_stage0(self):
384 """Fetch the build system for Rust, written in Rust
386 This method will build a cache directory, then it will fetch the
387 tarball which has the stage0 compiler used to then bootstrap the Rust
390 Each downloaded tarball is extracted, after that, the script
391 will move all the content to the right place.
393 rustc_channel = self.rustc_channel
394 rustfmt_channel = self.rustfmt_channel
396 if self.rustc().startswith(self.bin_root()) and \
397 (not os.path.exists(self.rustc()) or
398 self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
399 if os.path.exists(self.bin_root()):
400 shutil.rmtree(self.bin_root())
401 download_rustc = self.rustc_commit is not None
402 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
403 filename = "rust-std-{}-{}{}".format(
404 rustc_channel, self.build, tarball_suffix)
405 pattern = "rust-std-{}".format(self.build)
406 self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
407 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
409 self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
410 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
412 self._download_component_helper(filename, "cargo", tarball_suffix)
413 if self.rustc_commit is not None:
414 filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
415 self._download_component_helper(
416 filename, "rustc-dev", tarball_suffix, download_rustc
419 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
420 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
421 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
422 lib_dir = "{}/lib".format(self.bin_root())
423 for lib in os.listdir(lib_dir):
424 if lib.endswith(".so"):
425 self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
426 with output(self.rustc_stamp()) as rust_stamp:
427 rust_stamp.write(self.date + str(self.rustc_commit))
429 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
430 not os.path.exists(self.rustfmt())
431 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
434 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
435 [channel, date] = rustfmt_channel.split('-', 1)
436 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
437 self._download_component_helper(
438 filename, "rustfmt-preview", tarball_suffix, key=date
440 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
441 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
442 with output(self.rustfmt_stamp()) as rustfmt_stamp:
443 rustfmt_stamp.write(self.rustfmt_channel)
445 if self.downloading_llvm():
446 # We want the most recent LLVM submodule update to avoid downloading
447 # LLVM more often than necessary.
449 # This git command finds that commit SHA, looking for bors-authored
450 # merges that modified src/llvm-project.
452 # This works even in a repository that has not yet initialized
454 top_level = subprocess.check_output([
455 "git", "rev-parse", "--show-toplevel",
456 ]).decode(sys.getdefaultencoding()).strip()
457 llvm_sha = subprocess.check_output([
458 "git", "log", "--author=bors", "--format=%H", "-n1",
459 "-m", "--first-parent",
461 "{}/src/llvm-project".format(top_level),
462 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
463 ]).decode(sys.getdefaultencoding()).strip()
464 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
465 llvm_root = self.llvm_root()
466 llvm_lib = os.path.join(llvm_root, "lib")
467 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
468 self._download_ci_llvm(llvm_sha, llvm_assertions)
469 for binary in ["llvm-config", "FileCheck"]:
470 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
471 for lib in os.listdir(llvm_lib):
472 if lib.endswith(".so"):
473 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
474 with output(self.llvm_stamp()) as llvm_stamp:
475 llvm_stamp.write(llvm_sha + str(llvm_assertions))
477 def downloading_llvm(self):
478 opt = self.get_toml('download-ci-llvm', 'llvm')
479 # This is currently all tier 1 targets (since others may not have CI
481 # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
482 supported_platforms = [
483 "aarch64-unknown-linux-gnu",
484 "i686-pc-windows-gnu",
485 "i686-pc-windows-msvc",
486 "i686-unknown-linux-gnu",
487 "x86_64-unknown-linux-gnu",
488 "x86_64-apple-darwin",
489 "x86_64-pc-windows-gnu",
490 "x86_64-pc-windows-msvc",
492 return opt == "true" \
493 or (opt == "if-available" and self.build in supported_platforms)
495 def _download_component_helper(
496 self, filename, pattern, tarball_suffix, download_rustc=False, key=None
500 key = self.rustc_commit
503 cache_dst = os.path.join(self.build_dir, "cache")
504 rustc_cache = os.path.join(cache_dst, key)
505 if not os.path.exists(rustc_cache):
506 os.makedirs(rustc_cache)
509 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
511 url = "{}/dist/{}".format(self._download_url, key)
512 tarball = os.path.join(rustc_cache, filename)
513 if not os.path.exists(tarball):
514 do_verify = not download_rustc
515 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
516 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
518 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
519 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
520 cache_dst = os.path.join(self.build_dir, "cache")
521 rustc_cache = os.path.join(cache_dst, cache_prefix)
522 if not os.path.exists(rustc_cache):
523 os.makedirs(rustc_cache)
525 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
527 url = url.replace('rustc-builds', 'rustc-builds-alt')
528 # ci-artifacts are only stored as .xz, not .gz
530 print("error: XZ support is required to download LLVM")
531 print("help: consider disabling `download-ci-llvm` or using python3")
533 tarball_suffix = '.tar.xz'
534 filename = "rust-dev-nightly-" + self.build + tarball_suffix
535 tarball = os.path.join(rustc_cache, filename)
536 if not os.path.exists(tarball):
537 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
538 unpack(tarball, tarball_suffix, self.llvm_root(),
540 verbose=self.verbose)
542 def fix_bin_or_dylib(self, fname, rpath_libz=False):
543 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
544 or the RPATH section, to fix the dynamic library search path
546 This method is only required on NixOS and uses the PatchELF utility to
547 change the interpreter/RPATH of ELF executables.
549 Please see https://nixos.org/patchelf.html for more information
551 default_encoding = sys.getdefaultencoding()
553 ostype = subprocess.check_output(
554 ['uname', '-s']).strip().decode(default_encoding)
555 except subprocess.CalledProcessError:
557 except OSError as reason:
558 if getattr(reason, 'winerror', None) is not None:
562 if ostype != "Linux":
565 if not os.path.exists("/etc/NIXOS"):
567 if os.path.exists("/lib"):
570 # At this point we're pretty sure the user is running NixOS
571 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
572 print(nix_os_msg, fname)
574 # Only build `stage0/.nix-deps` once.
575 nix_deps_dir = self.nix_deps_dir
577 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
578 if not os.path.exists(nix_deps_dir):
579 os.makedirs(nix_deps_dir)
582 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
583 "stdenv.cc.bintools",
585 # Needed as a system dependency of `libLLVM-*.so`.
588 # Needed for patching ELF binaries (see doc comment above).
592 # Run `nix-build` to "build" each dependency (which will likely reuse
593 # the existing `/nix/store` copy, or at most download a pre-built copy).
594 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
595 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
596 # ensuring garbage collection will never remove the `/nix/store` path
597 # (which would break our patched binaries that hardcode those paths).
600 subprocess.check_output([
601 "nix-build", "<nixpkgs>",
603 "-o", "{}/{}".format(nix_deps_dir, dep),
605 except subprocess.CalledProcessError as reason:
606 print("warning: failed to call nix-build:", reason)
609 self.nix_deps_dir = nix_deps_dir
611 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
615 # Patch RPATH to add `zlib` dependency that stems from LLVM
616 dylib_deps = ["zlib"]
618 # Relative default, all binary and dynamic libraries we ship
619 # appear to have this (even when `../lib` is redundant).
621 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
622 patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
623 if not fname.endswith(".so"):
624 # Finally, set the corret .interp for binaries
625 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
626 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
627 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
630 subprocess.check_output([patchelf] + patchelf_args + [fname])
631 except subprocess.CalledProcessError as reason:
632 print("warning: failed to call patchelf:", reason)
635 # Return the stage1 compiler to download, if any.
636 def maybe_download_rustc(self):
637 # If `download-rustc` is not set, default to rebuilding.
638 if self.get_toml("download-rustc", section="rust") != "true":
641 # Handle running from a directory other than the top level
642 rev_parse = ["git", "rev-parse", "--show-toplevel"]
643 top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
644 compiler = "{}/compiler/".format(top_level)
646 # Look for a version to compare to based on the current commit.
647 # Ideally this would just use `merge-base`, but on beta and stable branches that wouldn't
648 # come up with any commits, so hack it and use `author=bors` instead.
649 merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1", "--", compiler]
650 commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
652 # Warn if there were changes to the compiler since the ancestor commit.
653 status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler])
655 print("warning: `download-rustc` is enabled, but there are changes to compiler/")
659 def rustc_stamp(self):
660 """Return the path for .rustc-stamp
663 >>> rb.build_dir = "build"
664 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
667 return os.path.join(self.bin_root(), '.rustc-stamp')
669 def rustfmt_stamp(self):
670 """Return the path for .rustfmt-stamp
673 >>> rb.build_dir = "build"
674 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
677 return os.path.join(self.bin_root(), '.rustfmt-stamp')
679 def llvm_stamp(self):
680 """Return the path for .rustfmt-stamp
683 >>> rb.build_dir = "build"
684 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
687 return os.path.join(self.llvm_root(), '.llvm-stamp')
690 def program_out_of_date(self, stamp_path, key):
691 """Check if the given program stamp is out of date"""
692 if not os.path.exists(stamp_path) or self.clean:
694 with open(stamp_path, 'r') as stamp:
695 return key != stamp.read()
698 """Return the binary root directory
701 >>> rb.build_dir = "build"
702 >>> rb.bin_root() == os.path.join("build", "stage0")
705 When the 'build' property is given should be a nested directory:
707 >>> rb.build = "devel"
708 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
711 return os.path.join(self.build_dir, self.build, "stage0")
714 """Return the CI LLVM root directory
717 >>> rb.build_dir = "build"
718 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
721 When the 'build' property is given should be a nested directory:
723 >>> rb.build = "devel"
724 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
727 return os.path.join(self.build_dir, self.build, "ci-llvm")
729 def get_toml(self, key, section=None):
730 """Returns the value of the given key in config.toml, otherwise returns None
733 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
734 >>> rb.get_toml("key2")
737 If the key does not exists, the result is None:
739 >>> rb.get_toml("key3") is None
742 Optionally also matches the section the key appears in
744 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
745 >>> rb.get_toml('key', 'a')
747 >>> rb.get_toml('key', 'b')
749 >>> rb.get_toml('key', 'c') is None
752 >>> rb.config_toml = 'key1 = true'
753 >>> rb.get_toml("key1")
758 for line in self.config_toml.splitlines():
759 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
760 if section_match is not None:
761 cur_section = section_match.group(1)
763 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
764 if match is not None:
765 value = match.group(1)
766 if section is None or section == cur_section:
767 return self.get_string(value) or value.strip()
771 """Return config path for cargo"""
772 return self.program_config('cargo')
775 """Return config path for rustc"""
776 return self.program_config('rustc')
779 """Return config path for rustfmt"""
780 if not self.rustfmt_channel:
782 return self.program_config('rustfmt')
784 def program_config(self, program):
785 """Return config path for the given program
788 >>> rb.config_toml = 'rustc = "rustc"\\n'
789 >>> rb.program_config('rustc')
791 >>> rb.config_toml = ''
792 >>> cargo_path = rb.program_config('cargo')
793 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
797 config = self.get_toml(program)
799 return os.path.expanduser(config)
800 return os.path.join(self.bin_root(), "bin", "{}{}".format(
801 program, self.exe_suffix()))
804 def get_string(line):
805 """Return the value between double quotes
807 >>> RustBuild.get_string(' "devel" ')
809 >>> RustBuild.get_string(" 'devel' ")
811 >>> RustBuild.get_string('devel') is None
813 >>> RustBuild.get_string(' "devel ')
816 start = line.find('"')
818 end = start + 1 + line[start + 1:].find('"')
819 return line[start + 1:end]
820 start = line.find('\'')
822 end = start + 1 + line[start + 1:].find('\'')
823 return line[start + 1:end]
828 """Return a suffix for executables"""
829 if sys.platform == 'win32':
833 def bootstrap_binary(self):
834 """Return the path of the bootstrap binary
837 >>> rb.build_dir = "build"
838 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
839 ... "debug", "bootstrap")
842 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
844 def build_bootstrap(self):
845 """Build bootstrap"""
846 build_dir = os.path.join(self.build_dir, "bootstrap")
847 if self.clean and os.path.exists(build_dir):
848 shutil.rmtree(build_dir)
849 env = os.environ.copy()
850 # `CARGO_BUILD_TARGET` breaks bootstrap build.
851 # See also: <https://github.com/rust-lang/rust/issues/70208>.
852 if "CARGO_BUILD_TARGET" in env:
853 del env["CARGO_BUILD_TARGET"]
854 env["CARGO_TARGET_DIR"] = build_dir
855 env["RUSTC"] = self.rustc()
856 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
857 (os.pathsep + env["LD_LIBRARY_PATH"]) \
858 if "LD_LIBRARY_PATH" in env else ""
859 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
860 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
861 if "DYLD_LIBRARY_PATH" in env else ""
862 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
863 (os.pathsep + env["LIBRARY_PATH"]) \
864 if "LIBRARY_PATH" in env else ""
865 # preserve existing RUSTFLAGS
866 env.setdefault("RUSTFLAGS", "")
867 env["RUSTFLAGS"] += " -Cdebuginfo=2"
869 build_section = "target.{}".format(self.build)
871 if self.get_toml("crt-static", build_section) == "true":
872 target_features += ["+crt-static"]
873 elif self.get_toml("crt-static", build_section) == "false":
874 target_features += ["-crt-static"]
876 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
877 target_linker = self.get_toml("linker", build_section)
878 if target_linker is not None:
879 env["RUSTFLAGS"] += " -C linker=" + target_linker
880 # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump
881 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
882 if self.get_toml("deny-warnings", "rust") != "false":
883 env["RUSTFLAGS"] += " -Dwarnings"
885 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
886 os.pathsep + env["PATH"]
887 if not os.path.isfile(self.cargo()):
888 raise Exception("no cargo executable found at `{}`".format(
890 args = [self.cargo(), "build", "--manifest-path",
891 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
892 for _ in range(1, self.verbose):
893 args.append("--verbose")
894 if self.use_locked_deps:
895 args.append("--locked")
896 if self.use_vendored_sources:
897 args.append("--frozen")
898 run(args, env=env, verbose=self.verbose)
900 def build_triple(self):
901 """Build triple as in LLVM
903 Note that `default_build_triple` is moderately expensive,
904 so use `self.build` where possible.
906 config = self.get_toml('build')
909 return default_build_triple(self.verbose)
911 def check_submodule(self, module, slow_submodules):
912 if not slow_submodules:
913 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
914 cwd=os.path.join(self.rust_root, module),
915 stdout=subprocess.PIPE)
920 def update_submodule(self, module, checked_out, recorded_submodules):
921 module_path = os.path.join(self.rust_root, module)
923 if checked_out is not None:
924 default_encoding = sys.getdefaultencoding()
925 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
926 if recorded_submodules[module] == checked_out:
929 print("Updating submodule", module)
931 run(["git", "submodule", "-q", "sync", module],
932 cwd=self.rust_root, verbose=self.verbose)
934 update_args = ["git", "submodule", "update", "--init", "--recursive"]
935 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
936 update_args.append("--progress")
937 update_args.append(module)
938 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
940 run(["git", "reset", "-q", "--hard"],
941 cwd=module_path, verbose=self.verbose)
942 run(["git", "clean", "-qdfx"],
943 cwd=module_path, verbose=self.verbose)
945 def update_submodules(self):
946 """Update submodules"""
947 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
948 self.get_toml('submodules') == "false":
951 default_encoding = sys.getdefaultencoding()
953 # check the existence and version of 'git' command
954 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
955 self.git_version = distutils.version.LooseVersion(git_version_str)
957 slow_submodules = self.get_toml('fast-submodules') == "false"
960 print('Unconditionally updating all submodules')
962 print('Updating only changed submodules')
963 default_encoding = sys.getdefaultencoding()
964 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
965 ["git", "config", "--file",
966 os.path.join(self.rust_root, ".gitmodules"),
967 "--get-regexp", "path"]
968 ).decode(default_encoding).splitlines()]
969 filtered_submodules = []
970 submodules_names = []
971 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
972 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
973 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
974 or "llvm" in self.get_toml('codegen-backends', 'rust')
975 for module in submodules:
976 if module.endswith("llvm-project"):
977 # Don't sync the llvm-project submodule if an external LLVM was
978 # provided, if we are downloading LLVM or if the LLVM backend is
979 # not being built. Also, if the submodule has been initialized
980 # already, sync it anyways so that it doesn't mess up contributor
982 if external_llvm_provided or not llvm_needed:
983 if self.get_toml('lld') != 'true' and not llvm_checked_out:
985 check = self.check_submodule(module, slow_submodules)
986 filtered_submodules.append((module, check))
987 submodules_names.append(module)
988 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
989 cwd=self.rust_root, stdout=subprocess.PIPE)
990 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
991 recorded_submodules = {}
992 for data in recorded:
994 recorded_submodules[data[3]] = data[2]
995 for module in filtered_submodules:
996 self.update_submodule(module[0], module[1], recorded_submodules)
997 print("Submodules updated in %.2f seconds" % (time() - start_time))
999 def set_normal_environment(self):
1000 """Set download URL for normal environment"""
1001 if 'RUSTUP_DIST_SERVER' in os.environ:
1002 self._download_url = os.environ['RUSTUP_DIST_SERVER']
1004 self._download_url = 'https://static.rust-lang.org'
1006 def set_dev_environment(self):
1007 """Set download URL for development environment"""
1008 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
1009 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
1011 self._download_url = 'https://dev-static.rust-lang.org'
1013 def check_vendored_status(self):
1014 """Check that vendoring is configured properly"""
1015 vendor_dir = os.path.join(self.rust_root, 'vendor')
1016 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
1017 if os.environ.get('USER') != os.environ['SUDO_USER']:
1018 self.use_vendored_sources = True
1019 print('info: looks like you are running this command under `sudo`')
1020 print(' and so in order to preserve your $HOME this will now')
1021 print(' use vendored sources by default.')
1022 if not os.path.exists(vendor_dir):
1023 print('error: vendoring required, but vendor directory does not exist.')
1024 print(' Run `cargo vendor` without sudo to initialize the '
1025 'vendor directory.')
1026 raise Exception("{} not found".format(vendor_dir))
1028 if self.use_vendored_sources:
1029 if not os.path.exists('.cargo'):
1030 os.makedirs('.cargo')
1031 with output('.cargo/config') as cargo_config:
1033 "[source.crates-io]\n"
1034 "replace-with = 'vendored-sources'\n"
1035 "registry = 'https://example.com'\n"
1037 "[source.vendored-sources]\n"
1038 "directory = '{}/vendor'\n"
1039 .format(self.rust_root))
1041 if os.path.exists('.cargo'):
1042 shutil.rmtree('.cargo')
1044 def ensure_vendored(self):
1045 """Ensure that the vendored sources are available if needed"""
1046 vendor_dir = os.path.join(self.rust_root, 'vendor')
1047 # Note that this does not handle updating the vendored dependencies if
1048 # the rust git repository is updated. Normal development usually does
1049 # not use vendoring, so hopefully this isn't too much of a problem.
1050 if self.use_vendored_sources and not os.path.exists(vendor_dir):
1054 "--sync=./src/tools/rust-analyzer/Cargo.toml",
1055 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
1056 ], verbose=self.verbose, cwd=self.rust_root)
1059 def bootstrap(help_triggered):
1060 """Configure, fetch, build and run the initial bootstrap"""
1062 # If the user is asking for help, let them know that the whole download-and-build
1063 # process has to happen before anything is printed out.
1065 print("info: Downloading and building bootstrap before processing --help")
1066 print(" command. See src/bootstrap/README.md for help with common")
1069 parser = argparse.ArgumentParser(description='Build rust')
1070 parser.add_argument('--config')
1071 parser.add_argument('--build')
1072 parser.add_argument('--clean', action='store_true')
1073 parser.add_argument('-v', '--verbose', action='count', default=0)
1075 args = [a for a in sys.argv if a != '-h' and a != '--help']
1076 args, _ = parser.parse_known_args(args)
1078 # Configure initial bootstrap
1080 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1081 build.verbose = args.verbose
1082 build.clean = args.clean
1084 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1086 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1087 if not toml_path and os.path.exists('config.toml'):
1088 toml_path = 'config.toml'
1091 if not os.path.exists(toml_path):
1092 toml_path = os.path.join(build.rust_root, toml_path)
1094 with open(toml_path) as config:
1095 build.config_toml = config.read()
1097 profile = build.get_toml('profile')
1098 if profile is not None:
1099 include_file = 'config.{}.toml'.format(profile)
1100 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1101 include_path = os.path.join(include_dir, include_file)
1102 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1103 # specific key, so appending our defaults at the end allows the user to override them
1104 with open(include_path) as included_toml:
1105 build.config_toml += os.linesep + included_toml.read()
1107 config_verbose = build.get_toml('verbose', 'build')
1108 if config_verbose is not None:
1109 build.verbose = max(build.verbose, int(config_verbose))
1111 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1113 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1115 build.check_vendored_status()
1117 build_dir = build.get_toml('build-dir', 'build') or 'build'
1118 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1120 data = stage0_data(build.rust_root)
1121 build.date = data['date']
1122 build.rustc_channel = data['rustc']
1124 if "rustfmt" in data:
1125 build.rustfmt_channel = data['rustfmt']
1128 build.set_dev_environment()
1130 build.set_normal_environment()
1132 build.build = args.build or build.build_triple()
1133 build.update_submodules()
1135 # Fetch/build the bootstrap
1136 build.rustc_commit = build.maybe_download_rustc()
1137 if build.rustc_commit is not None:
1139 commit = build.rustc_commit
1140 print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
1141 # FIXME: support downloading artifacts from the beta channel
1142 build.rustc_channel = "nightly"
1143 build.download_stage0()
1145 build.ensure_vendored()
1146 build.build_bootstrap()
1150 args = [build.bootstrap_binary()]
1151 args.extend(sys.argv[1:])
1152 env = os.environ.copy()
1153 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1154 env["BOOTSTRAP_PYTHON"] = sys.executable
1155 env["BUILD_DIR"] = build.build_dir
1156 env["RUSTC_BOOTSTRAP"] = '1'
1158 env["BOOTSTRAP_CONFIG"] = toml_path
1159 run(args, env=env, verbose=build.verbose)
1163 """Entry point for the bootstrap process"""
1166 # x.py help <cmd> ...
1167 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1168 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1171 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1173 bootstrap(help_triggered)
1174 if not help_triggered:
1175 print("Build completed successfully in {}".format(
1176 format_build_time(time() - start_time)))
1177 except (SystemExit, KeyboardInterrupt) as error:
1178 if hasattr(error, 'code') and isinstance(error.code, int):
1179 exit_code = error.code
1183 if not help_triggered:
1184 print("Build completed unsuccessfully in {}".format(
1185 format_build_time(time() - start_time)))
1189 if __name__ == '__main__':