1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"])
198 version = version.decode(default_encoding)
199 host = next(x for x in version.split('\n') if x.startswith("host: "))
200 triple = host.split("host: ")[1]
202 print("detected default triple {}".format(triple))
204 except Exception as e:
206 print("rustup not detected: {}".format(e))
207 print("falling back to auto-detect")
209 required = sys.platform != 'win32'
210 ostype = require(["uname", "-s"], exit=required)
211 cputype = require(['uname', '-m'], exit=required)
213 # If we do not have `uname`, assume Windows.
214 if ostype is None or cputype is None:
215 return 'x86_64-pc-windows-msvc'
217 ostype = ostype.decode(default_encoding)
218 cputype = cputype.decode(default_encoding)
220 # The goal here is to come up with the same triple as LLVM would,
221 # at least for the subset of platforms we're willing to target.
223 'Darwin': 'apple-darwin',
224 'DragonFly': 'unknown-dragonfly',
225 'FreeBSD': 'unknown-freebsd',
226 'Haiku': 'unknown-haiku',
227 'NetBSD': 'unknown-netbsd',
228 'OpenBSD': 'unknown-openbsd'
231 # Consider the direct transformation first and then the special cases
232 if ostype in ostype_mapper:
233 ostype = ostype_mapper[ostype]
234 elif ostype == 'Linux':
235 os_from_sp = subprocess.check_output(
236 ['uname', '-o']).strip().decode(default_encoding)
237 if os_from_sp == 'Android':
238 ostype = 'linux-android'
240 ostype = 'unknown-linux-gnu'
241 elif ostype == 'SunOS':
242 ostype = 'sun-solaris'
243 # On Solaris, uname -m will return a machine classification instead
244 # of a cpu type, so uname -p is recommended instead. However, the
245 # output from that option is too generic for our purposes (it will
246 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
247 # must be used instead.
248 cputype = require(['isainfo', '-k']).decode(default_encoding)
249 elif ostype.startswith('MINGW'):
250 # msys' `uname` does not print gcc configuration, but prints msys
251 # configuration. so we cannot believe `uname -m`:
252 # msys1 is always i686 and msys2 is always x86_64.
253 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
255 ostype = 'pc-windows-gnu'
257 if os.environ.get('MSYSTEM') == 'MINGW64':
259 elif ostype.startswith('MSYS'):
260 ostype = 'pc-windows-gnu'
261 elif ostype.startswith('CYGWIN_NT'):
263 if ostype.endswith('WOW64'):
265 ostype = 'pc-windows-gnu'
266 elif sys.platform == 'win32':
267 # Some Windows platforms might have a `uname` command that returns a
268 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
269 # these cases, fall back to using sys.platform.
270 return 'x86_64-pc-windows-msvc'
272 err = "unknown OS type: {}".format(ostype)
275 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
276 cputype = subprocess.check_output(
277 ['uname', '-p']).strip().decode(default_encoding)
280 'aarch64': 'aarch64',
287 'powerpc': 'powerpc',
288 'powerpc64': 'powerpc64',
289 'powerpc64le': 'powerpc64le',
291 'ppc64': 'powerpc64',
292 'ppc64le': 'powerpc64le',
300 # Consider the direct transformation first and then the special cases
301 if cputype in cputype_mapper:
302 cputype = cputype_mapper[cputype]
303 elif cputype in {'xscale', 'arm'}:
305 if ostype == 'linux-android':
306 ostype = 'linux-androideabi'
307 elif ostype == 'unknown-freebsd':
308 cputype = subprocess.check_output(
309 ['uname', '-p']).strip().decode(default_encoding)
310 ostype = 'unknown-freebsd'
311 elif cputype == 'armv6l':
313 if ostype == 'linux-android':
314 ostype = 'linux-androideabi'
317 elif cputype in {'armv7l', 'armv8l'}:
319 if ostype == 'linux-android':
320 ostype = 'linux-androideabi'
323 elif cputype == 'mips':
324 if sys.byteorder == 'big':
326 elif sys.byteorder == 'little':
329 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
330 elif cputype == 'mips64':
331 if sys.byteorder == 'big':
333 elif sys.byteorder == 'little':
336 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
337 # only the n64 ABI is supported, indicate it
339 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
342 err = "unknown cpu type: {}".format(cputype)
345 return "{}-{}".format(cputype, ostype)
348 @contextlib.contextmanager
349 def output(filepath):
350 tmp = filepath + '.tmp'
351 with open(tmp, 'w') as f:
354 if os.path.exists(filepath):
355 os.remove(filepath) # PermissionError/OSError on Win32 if in use
357 shutil.copy2(tmp, filepath)
360 os.rename(tmp, filepath)
363 class RustBuild(object):
364 """Provide all the methods required to build Rust"""
367 self._download_url = ''
368 self.rustc_channel = ''
369 self.rustfmt_channel = ''
373 self.config_toml = ''
375 self.use_locked_deps = ''
376 self.use_vendored_sources = ''
378 self.git_version = None
379 self.nix_deps_dir = None
381 def download_stage0(self):
382 """Fetch the build system for Rust, written in Rust
384 This method will build a cache directory, then it will fetch the
385 tarball which has the stage0 compiler used to then bootstrap the Rust
388 Each downloaded tarball is extracted, after that, the script
389 will move all the content to the right place.
391 rustc_channel = self.rustc_channel
392 rustfmt_channel = self.rustfmt_channel
394 if self.rustc().startswith(self.bin_root()) and \
395 (not os.path.exists(self.rustc()) or
396 self.program_out_of_date(self.rustc_stamp(), self.date)):
397 if os.path.exists(self.bin_root()):
398 shutil.rmtree(self.bin_root())
399 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
400 filename = "rust-std-{}-{}{}".format(
401 rustc_channel, self.build, tarball_suffix)
402 pattern = "rust-std-{}".format(self.build)
403 self._download_stage0_helper(filename, pattern, tarball_suffix)
404 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
406 self._download_stage0_helper(filename, "rustc", tarball_suffix)
407 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
409 self._download_stage0_helper(filename, "cargo", tarball_suffix)
410 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
411 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
412 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
413 lib_dir = "{}/lib".format(self.bin_root())
414 for lib in os.listdir(lib_dir):
415 if lib.endswith(".so"):
416 self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
417 with output(self.rustc_stamp()) as rust_stamp:
418 rust_stamp.write(self.date)
420 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
421 not os.path.exists(self.rustfmt())
422 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
425 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
426 [channel, date] = rustfmt_channel.split('-', 1)
427 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
428 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
429 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
430 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
431 with output(self.rustfmt_stamp()) as rustfmt_stamp:
432 rustfmt_stamp.write(self.rustfmt_channel)
434 if self.downloading_llvm():
435 # We want the most recent LLVM submodule update to avoid downloading
436 # LLVM more often than necessary.
438 # This git command finds that commit SHA, looking for bors-authored
439 # merges that modified src/llvm-project.
441 # This works even in a repository that has not yet initialized
443 top_level = subprocess.check_output([
444 "git", "rev-parse", "--show-toplevel",
445 ]).decode(sys.getdefaultencoding()).strip()
446 llvm_sha = subprocess.check_output([
447 "git", "log", "--author=bors", "--format=%H", "-n1",
448 "-m", "--first-parent",
450 "{}/src/llvm-project".format(top_level),
451 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
452 ]).decode(sys.getdefaultencoding()).strip()
453 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
454 llvm_root = self.llvm_root()
455 llvm_lib = os.path.join(llvm_root, "lib")
456 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
457 self._download_ci_llvm(llvm_sha, llvm_assertions)
458 for binary in ["llvm-config", "FileCheck"]:
459 self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary), rpath_libz=True)
460 for lib in os.listdir(llvm_lib):
461 if lib.endswith(".so"):
462 self.fix_bin_or_dylib(os.path.join(llvm_lib, lib), rpath_libz=True)
463 with output(self.llvm_stamp()) as llvm_stamp:
464 llvm_stamp.write(llvm_sha + str(llvm_assertions))
466 def downloading_llvm(self):
467 opt = self.get_toml('download-ci-llvm', 'llvm')
468 return opt == "true" \
469 or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
471 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
474 cache_dst = os.path.join(self.build_dir, "cache")
475 rustc_cache = os.path.join(cache_dst, date)
476 if not os.path.exists(rustc_cache):
477 os.makedirs(rustc_cache)
479 url = "{}/dist/{}".format(self._download_url, date)
480 tarball = os.path.join(rustc_cache, filename)
481 if not os.path.exists(tarball):
482 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
483 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
485 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
486 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
487 cache_dst = os.path.join(self.build_dir, "cache")
488 rustc_cache = os.path.join(cache_dst, cache_prefix)
489 if not os.path.exists(rustc_cache):
490 os.makedirs(rustc_cache)
492 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
494 url = url.replace('rustc-builds', 'rustc-builds-alt')
495 # ci-artifacts are only stored as .xz, not .gz
497 print("error: XZ support is required to download LLVM")
498 print("help: consider disabling `download-ci-llvm` or using python3")
500 tarball_suffix = '.tar.xz'
501 filename = "rust-dev-nightly-" + self.build + tarball_suffix
502 tarball = os.path.join(rustc_cache, filename)
503 if not os.path.exists(tarball):
504 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
505 unpack(tarball, tarball_suffix, self.llvm_root(),
507 verbose=self.verbose)
509 def fix_bin_or_dylib(self, fname, rpath_libz=False):
510 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
511 or the RPATH section, to fix the dynamic library search path
513 This method is only required on NixOS and uses the PatchELF utility to
514 change the interpreter/RPATH of ELF executables.
516 Please see https://nixos.org/patchelf.html for more information
518 default_encoding = sys.getdefaultencoding()
520 ostype = subprocess.check_output(
521 ['uname', '-s']).strip().decode(default_encoding)
522 except subprocess.CalledProcessError:
524 except OSError as reason:
525 if getattr(reason, 'winerror', None) is not None:
529 if ostype != "Linux":
532 if not os.path.exists("/etc/NIXOS"):
534 if os.path.exists("/lib"):
537 # At this point we're pretty sure the user is running NixOS
538 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
539 print(nix_os_msg, fname)
541 # Only build `stage0/.nix-deps` once.
542 nix_deps_dir = self.nix_deps_dir
544 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
545 if not os.path.exists(nix_deps_dir):
546 os.makedirs(nix_deps_dir)
549 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
550 "stdenv.cc.bintools",
552 # Needed as a system dependency of `libLLVM-*.so`.
555 # Needed for patching ELF binaries (see doc comment above).
559 # Run `nix-build` to "build" each dependency (which will likely reuse
560 # the existing `/nix/store` copy, or at most download a pre-built copy).
561 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
562 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
563 # ensuring garbage collection will never remove the `/nix/store` path
564 # (which would break our patched binaries that hardcode those paths).
567 subprocess.check_output([
568 "nix-build", "<nixpkgs>",
570 "-o", "{}/{}".format(nix_deps_dir, dep),
572 except subprocess.CalledProcessError as reason:
573 print("warning: failed to call nix-build:", reason)
576 self.nix_deps_dir = nix_deps_dir
578 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
582 # Patch RPATH to add `zlib` dependency that stems from LLVM
583 dylib_deps = ["zlib"]
585 # Relative default, all binary and dynamic libraries we ship
586 # appear to have this (even when `../lib` is redundant).
588 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
589 patchelf_args += ["--set-rpath", ":".join(rpath_entries)]
590 if not fname.endswith(".so"):
591 # Finally, set the corret .interp for binaries
592 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
593 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
594 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
597 subprocess.check_output([patchelf] + patchelf_args + [fname])
598 except subprocess.CalledProcessError as reason:
599 print("warning: failed to call patchelf:", reason)
602 def rustc_stamp(self):
603 """Return the path for .rustc-stamp
606 >>> rb.build_dir = "build"
607 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
610 return os.path.join(self.bin_root(), '.rustc-stamp')
612 def rustfmt_stamp(self):
613 """Return the path for .rustfmt-stamp
616 >>> rb.build_dir = "build"
617 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
620 return os.path.join(self.bin_root(), '.rustfmt-stamp')
622 def llvm_stamp(self):
623 """Return the path for .rustfmt-stamp
626 >>> rb.build_dir = "build"
627 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
630 return os.path.join(self.llvm_root(), '.llvm-stamp')
633 def program_out_of_date(self, stamp_path, key):
634 """Check if the given program stamp is out of date"""
635 if not os.path.exists(stamp_path) or self.clean:
637 with open(stamp_path, 'r') as stamp:
638 return key != stamp.read()
641 """Return the binary root directory
644 >>> rb.build_dir = "build"
645 >>> rb.bin_root() == os.path.join("build", "stage0")
648 When the 'build' property is given should be a nested directory:
650 >>> rb.build = "devel"
651 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
654 return os.path.join(self.build_dir, self.build, "stage0")
657 """Return the CI LLVM root directory
660 >>> rb.build_dir = "build"
661 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
664 When the 'build' property is given should be a nested directory:
666 >>> rb.build = "devel"
667 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
670 return os.path.join(self.build_dir, self.build, "ci-llvm")
672 def get_toml(self, key, section=None):
673 """Returns the value of the given key in config.toml, otherwise returns None
676 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
677 >>> rb.get_toml("key2")
680 If the key does not exists, the result is None:
682 >>> rb.get_toml("key3") is None
685 Optionally also matches the section the key appears in
687 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
688 >>> rb.get_toml('key', 'a')
690 >>> rb.get_toml('key', 'b')
692 >>> rb.get_toml('key', 'c') is None
695 >>> rb.config_toml = 'key1 = true'
696 >>> rb.get_toml("key1")
701 for line in self.config_toml.splitlines():
702 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
703 if section_match is not None:
704 cur_section = section_match.group(1)
706 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
707 if match is not None:
708 value = match.group(1)
709 if section is None or section == cur_section:
710 return self.get_string(value) or value.strip()
714 """Return config path for cargo"""
715 return self.program_config('cargo')
718 """Return config path for rustc"""
719 return self.program_config('rustc')
722 """Return config path for rustfmt"""
723 if not self.rustfmt_channel:
725 return self.program_config('rustfmt')
727 def program_config(self, program):
728 """Return config path for the given program
731 >>> rb.config_toml = 'rustc = "rustc"\\n'
732 >>> rb.program_config('rustc')
734 >>> rb.config_toml = ''
735 >>> cargo_path = rb.program_config('cargo')
736 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
740 config = self.get_toml(program)
742 return os.path.expanduser(config)
743 return os.path.join(self.bin_root(), "bin", "{}{}".format(
744 program, self.exe_suffix()))
747 def get_string(line):
748 """Return the value between double quotes
750 >>> RustBuild.get_string(' "devel" ')
752 >>> RustBuild.get_string(" 'devel' ")
754 >>> RustBuild.get_string('devel') is None
756 >>> RustBuild.get_string(' "devel ')
759 start = line.find('"')
761 end = start + 1 + line[start + 1:].find('"')
762 return line[start + 1:end]
763 start = line.find('\'')
765 end = start + 1 + line[start + 1:].find('\'')
766 return line[start + 1:end]
771 """Return a suffix for executables"""
772 if sys.platform == 'win32':
776 def bootstrap_binary(self):
777 """Return the path of the bootstrap binary
780 >>> rb.build_dir = "build"
781 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
782 ... "debug", "bootstrap")
785 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
787 def build_bootstrap(self):
788 """Build bootstrap"""
789 build_dir = os.path.join(self.build_dir, "bootstrap")
790 if self.clean and os.path.exists(build_dir):
791 shutil.rmtree(build_dir)
792 env = os.environ.copy()
793 # `CARGO_BUILD_TARGET` breaks bootstrap build.
794 # See also: <https://github.com/rust-lang/rust/issues/70208>.
795 if "CARGO_BUILD_TARGET" in env:
796 del env["CARGO_BUILD_TARGET"]
797 env["CARGO_TARGET_DIR"] = build_dir
798 env["RUSTC"] = self.rustc()
799 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
800 (os.pathsep + env["LD_LIBRARY_PATH"]) \
801 if "LD_LIBRARY_PATH" in env else ""
802 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
803 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
804 if "DYLD_LIBRARY_PATH" in env else ""
805 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
806 (os.pathsep + env["LIBRARY_PATH"]) \
807 if "LIBRARY_PATH" in env else ""
808 # preserve existing RUSTFLAGS
809 env.setdefault("RUSTFLAGS", "")
810 env["RUSTFLAGS"] += " -Cdebuginfo=2"
812 build_section = "target.{}".format(self.build)
814 if self.get_toml("crt-static", build_section) == "true":
815 target_features += ["+crt-static"]
816 elif self.get_toml("crt-static", build_section) == "false":
817 target_features += ["-crt-static"]
819 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
820 target_linker = self.get_toml("linker", build_section)
821 if target_linker is not None:
822 env["RUSTFLAGS"] += " -C linker=" + target_linker
823 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
824 if self.get_toml("deny-warnings", "rust") != "false":
825 env["RUSTFLAGS"] += " -Dwarnings"
827 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
828 os.pathsep + env["PATH"]
829 if not os.path.isfile(self.cargo()):
830 raise Exception("no cargo executable found at `{}`".format(
832 args = [self.cargo(), "build", "--manifest-path",
833 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
834 for _ in range(1, self.verbose):
835 args.append("--verbose")
836 if self.use_locked_deps:
837 args.append("--locked")
838 if self.use_vendored_sources:
839 args.append("--frozen")
840 run(args, env=env, verbose=self.verbose)
842 def build_triple(self):
843 """Build triple as in LLVM
845 Note that `default_build_triple` is moderately expensive,
846 so use `self.build` where possible.
848 config = self.get_toml('build')
851 return default_build_triple(self.verbose)
853 def check_submodule(self, module, slow_submodules):
854 if not slow_submodules:
855 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
856 cwd=os.path.join(self.rust_root, module),
857 stdout=subprocess.PIPE)
862 def update_submodule(self, module, checked_out, recorded_submodules):
863 module_path = os.path.join(self.rust_root, module)
865 if checked_out is not None:
866 default_encoding = sys.getdefaultencoding()
867 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
868 if recorded_submodules[module] == checked_out:
871 print("Updating submodule", module)
873 run(["git", "submodule", "-q", "sync", module],
874 cwd=self.rust_root, verbose=self.verbose)
876 update_args = ["git", "submodule", "update", "--init", "--recursive"]
877 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
878 update_args.append("--progress")
879 update_args.append(module)
880 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
882 run(["git", "reset", "-q", "--hard"],
883 cwd=module_path, verbose=self.verbose)
884 run(["git", "clean", "-qdfx"],
885 cwd=module_path, verbose=self.verbose)
887 def update_submodules(self):
888 """Update submodules"""
889 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
890 self.get_toml('submodules') == "false":
893 default_encoding = sys.getdefaultencoding()
895 # check the existence and version of 'git' command
896 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
897 self.git_version = distutils.version.LooseVersion(git_version_str)
899 slow_submodules = self.get_toml('fast-submodules') == "false"
902 print('Unconditionally updating all submodules')
904 print('Updating only changed submodules')
905 default_encoding = sys.getdefaultencoding()
906 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
907 ["git", "config", "--file",
908 os.path.join(self.rust_root, ".gitmodules"),
909 "--get-regexp", "path"]
910 ).decode(default_encoding).splitlines()]
911 filtered_submodules = []
912 submodules_names = []
913 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
914 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
915 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
916 or "llvm" in self.get_toml('codegen-backends', 'rust')
917 for module in submodules:
918 if module.endswith("llvm-project"):
919 # Don't sync the llvm-project submodule if an external LLVM was
920 # provided, if we are downloading LLVM or if the LLVM backend is
921 # not being built. Also, if the submodule has been initialized
922 # already, sync it anyways so that it doesn't mess up contributor
924 if external_llvm_provided or not llvm_needed:
925 if self.get_toml('lld') != 'true' and not llvm_checked_out:
927 check = self.check_submodule(module, slow_submodules)
928 filtered_submodules.append((module, check))
929 submodules_names.append(module)
930 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
931 cwd=self.rust_root, stdout=subprocess.PIPE)
932 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
933 recorded_submodules = {}
934 for data in recorded:
936 recorded_submodules[data[3]] = data[2]
937 for module in filtered_submodules:
938 self.update_submodule(module[0], module[1], recorded_submodules)
939 print("Submodules updated in %.2f seconds" % (time() - start_time))
941 def set_normal_environment(self):
942 """Set download URL for normal environment"""
943 if 'RUSTUP_DIST_SERVER' in os.environ:
944 self._download_url = os.environ['RUSTUP_DIST_SERVER']
946 self._download_url = 'https://static.rust-lang.org'
948 def set_dev_environment(self):
949 """Set download URL for development environment"""
950 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
951 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
953 self._download_url = 'https://dev-static.rust-lang.org'
955 def check_vendored_status(self):
956 """Check that vendoring is configured properly"""
957 vendor_dir = os.path.join(self.rust_root, 'vendor')
958 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
959 if os.environ.get('USER') != os.environ['SUDO_USER']:
960 self.use_vendored_sources = True
961 print('info: looks like you are running this command under `sudo`')
962 print(' and so in order to preserve your $HOME this will now')
963 print(' use vendored sources by default.')
964 if not os.path.exists(vendor_dir):
965 print('error: vendoring required, but vendor directory does not exist.')
966 print(' Run `cargo vendor` without sudo to initialize the '
968 raise Exception("{} not found".format(vendor_dir))
970 if self.use_vendored_sources:
971 if not os.path.exists('.cargo'):
972 os.makedirs('.cargo')
973 with output('.cargo/config') as cargo_config:
975 "[source.crates-io]\n"
976 "replace-with = 'vendored-sources'\n"
977 "registry = 'https://example.com'\n"
979 "[source.vendored-sources]\n"
980 "directory = '{}/vendor'\n"
981 .format(self.rust_root))
983 if os.path.exists('.cargo'):
984 shutil.rmtree('.cargo')
986 def ensure_vendored(self):
987 """Ensure that the vendored sources are available if needed"""
988 vendor_dir = os.path.join(self.rust_root, 'vendor')
989 # Note that this does not handle updating the vendored dependencies if
990 # the rust git repository is updated. Normal development usually does
991 # not use vendoring, so hopefully this isn't too much of a problem.
992 if self.use_vendored_sources and not os.path.exists(vendor_dir):
996 "--sync=./src/tools/rust-analyzer/Cargo.toml",
997 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
998 ], verbose=self.verbose, cwd=self.rust_root)
1001 def bootstrap(help_triggered):
1002 """Configure, fetch, build and run the initial bootstrap"""
1004 # If the user is asking for help, let them know that the whole download-and-build
1005 # process has to happen before anything is printed out.
1007 print("info: Downloading and building bootstrap before processing --help")
1008 print(" command. See src/bootstrap/README.md for help with common")
1011 parser = argparse.ArgumentParser(description='Build rust')
1012 parser.add_argument('--config')
1013 parser.add_argument('--build')
1014 parser.add_argument('--clean', action='store_true')
1015 parser.add_argument('-v', '--verbose', action='count', default=0)
1017 args = [a for a in sys.argv if a != '-h' and a != '--help']
1018 args, _ = parser.parse_known_args(args)
1020 # Configure initial bootstrap
1022 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1023 build.verbose = args.verbose
1024 build.clean = args.clean
1026 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1028 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1029 if not toml_path and os.path.exists('config.toml'):
1030 toml_path = 'config.toml'
1033 if not os.path.exists(toml_path):
1034 toml_path = os.path.join(build.rust_root, toml_path)
1036 with open(toml_path) as config:
1037 build.config_toml = config.read()
1039 profile = build.get_toml('profile')
1040 if profile is not None:
1041 include_file = 'config.{}.toml'.format(profile)
1042 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1043 include_path = os.path.join(include_dir, include_file)
1044 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1045 # specific key, so appending our defaults at the end allows the user to override them
1046 with open(include_path) as included_toml:
1047 build.config_toml += os.linesep + included_toml.read()
1049 config_verbose = build.get_toml('verbose', 'build')
1050 if config_verbose is not None:
1051 build.verbose = max(build.verbose, int(config_verbose))
1053 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1055 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1057 build.check_vendored_status()
1059 build_dir = build.get_toml('build-dir', 'build') or 'build'
1060 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1062 data = stage0_data(build.rust_root)
1063 build.date = data['date']
1064 build.rustc_channel = data['rustc']
1066 if "rustfmt" in data:
1067 build.rustfmt_channel = data['rustfmt']
1070 build.set_dev_environment()
1072 build.set_normal_environment()
1074 build.update_submodules()
1076 # Fetch/build the bootstrap
1077 build.build = args.build or build.build_triple()
1078 build.download_stage0()
1080 build.ensure_vendored()
1081 build.build_bootstrap()
1085 args = [build.bootstrap_binary()]
1086 args.extend(sys.argv[1:])
1087 env = os.environ.copy()
1088 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1089 env["BOOTSTRAP_PYTHON"] = sys.executable
1090 env["BUILD_DIR"] = build.build_dir
1091 env["RUSTC_BOOTSTRAP"] = '1'
1093 env["BOOTSTRAP_CONFIG"] = toml_path
1094 run(args, env=env, verbose=build.verbose)
1098 """Entry point for the bootstrap process"""
1101 # x.py help <cmd> ...
1102 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1103 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1106 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1108 bootstrap(help_triggered)
1109 if not help_triggered:
1110 print("Build completed successfully in {}".format(
1111 format_build_time(time() - start_time)))
1112 except (SystemExit, KeyboardInterrupt) as error:
1113 if hasattr(error, 'code') and isinstance(error.code, int):
1114 exit_code = error.code
1118 if not help_triggered:
1119 print("Build completed unsuccessfully in {}".format(
1120 format_build_time(time() - start_time)))
1124 if __name__ == '__main__':