1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"])
198 version = version.decode(default_encoding)
199 host = next(x for x in version.split('\n') if x.startswith("host: "))
200 triple = host.split("host: ")[1]
202 print("detected default triple {}".format(triple))
204 except Exception as e:
206 print("rustup not detected: {}".format(e))
207 print("falling back to auto-detect")
209 required = sys.platform != 'win32'
210 ostype = require(["uname", "-s"], exit=required)
211 cputype = require(['uname', '-m'], exit=required)
213 # If we do not have `uname`, assume Windows.
214 if ostype is None or cputype is None:
215 return 'x86_64-pc-windows-msvc'
217 ostype = ostype.decode(default_encoding)
218 cputype = cputype.decode(default_encoding)
220 # The goal here is to come up with the same triple as LLVM would,
221 # at least for the subset of platforms we're willing to target.
223 'Darwin': 'apple-darwin',
224 'DragonFly': 'unknown-dragonfly',
225 'FreeBSD': 'unknown-freebsd',
226 'Haiku': 'unknown-haiku',
227 'NetBSD': 'unknown-netbsd',
228 'OpenBSD': 'unknown-openbsd'
231 # Consider the direct transformation first and then the special cases
232 if ostype in ostype_mapper:
233 ostype = ostype_mapper[ostype]
234 elif ostype == 'Linux':
235 os_from_sp = subprocess.check_output(
236 ['uname', '-o']).strip().decode(default_encoding)
237 if os_from_sp == 'Android':
238 ostype = 'linux-android'
240 ostype = 'unknown-linux-gnu'
241 elif ostype == 'SunOS':
242 ostype = 'sun-solaris'
243 # On Solaris, uname -m will return a machine classification instead
244 # of a cpu type, so uname -p is recommended instead. However, the
245 # output from that option is too generic for our purposes (it will
246 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
247 # must be used instead.
248 cputype = require(['isainfo', '-k']).decode(default_encoding)
249 elif ostype.startswith('MINGW'):
250 # msys' `uname` does not print gcc configuration, but prints msys
251 # configuration. so we cannot believe `uname -m`:
252 # msys1 is always i686 and msys2 is always x86_64.
253 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
255 ostype = 'pc-windows-gnu'
257 if os.environ.get('MSYSTEM') == 'MINGW64':
259 elif ostype.startswith('MSYS'):
260 ostype = 'pc-windows-gnu'
261 elif ostype.startswith('CYGWIN_NT'):
263 if ostype.endswith('WOW64'):
265 ostype = 'pc-windows-gnu'
266 elif sys.platform == 'win32':
267 # Some Windows platforms might have a `uname` command that returns a
268 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
269 # these cases, fall back to using sys.platform.
270 return 'x86_64-pc-windows-msvc'
272 err = "unknown OS type: {}".format(ostype)
275 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
276 cputype = subprocess.check_output(
277 ['uname', '-p']).strip().decode(default_encoding)
280 'aarch64': 'aarch64',
287 'powerpc': 'powerpc',
288 'powerpc64': 'powerpc64',
289 'powerpc64le': 'powerpc64le',
291 'ppc64': 'powerpc64',
292 'ppc64le': 'powerpc64le',
300 # Consider the direct transformation first and then the special cases
301 if cputype in cputype_mapper:
302 cputype = cputype_mapper[cputype]
303 elif cputype in {'xscale', 'arm'}:
305 if ostype == 'linux-android':
306 ostype = 'linux-androideabi'
307 elif ostype == 'unknown-freebsd':
308 cputype = subprocess.check_output(
309 ['uname', '-p']).strip().decode(default_encoding)
310 ostype = 'unknown-freebsd'
311 elif cputype == 'armv6l':
313 if ostype == 'linux-android':
314 ostype = 'linux-androideabi'
317 elif cputype in {'armv7l', 'armv8l'}:
319 if ostype == 'linux-android':
320 ostype = 'linux-androideabi'
323 elif cputype == 'mips':
324 if sys.byteorder == 'big':
326 elif sys.byteorder == 'little':
329 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
330 elif cputype == 'mips64':
331 if sys.byteorder == 'big':
333 elif sys.byteorder == 'little':
336 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
337 # only the n64 ABI is supported, indicate it
339 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
342 err = "unknown cpu type: {}".format(cputype)
345 return "{}-{}".format(cputype, ostype)
348 @contextlib.contextmanager
349 def output(filepath):
350 tmp = filepath + '.tmp'
351 with open(tmp, 'w') as f:
354 if os.path.exists(filepath):
355 os.remove(filepath) # PermissionError/OSError on Win32 if in use
357 shutil.copy2(tmp, filepath)
360 os.rename(tmp, filepath)
363 class RustBuild(object):
364 """Provide all the methods required to build Rust"""
367 self._download_url = ''
368 self.rustc_channel = ''
369 self.rustfmt_channel = ''
373 self.config_toml = ''
375 self.use_locked_deps = ''
376 self.use_vendored_sources = ''
378 self.git_version = None
379 self.nix_deps_dir = None
381 def download_stage0(self):
382 """Fetch the build system for Rust, written in Rust
384 This method will build a cache directory, then it will fetch the
385 tarball which has the stage0 compiler used to then bootstrap the Rust
388 Each downloaded tarball is extracted, after that, the script
389 will move all the content to the right place.
391 rustc_channel = self.rustc_channel
392 rustfmt_channel = self.rustfmt_channel
394 if self.rustc().startswith(self.bin_root()) and \
395 (not os.path.exists(self.rustc()) or
396 self.program_out_of_date(self.rustc_stamp())):
397 if os.path.exists(self.bin_root()):
398 shutil.rmtree(self.bin_root())
399 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
400 filename = "rust-std-{}-{}{}".format(
401 rustc_channel, self.build, tarball_suffix)
402 pattern = "rust-std-{}".format(self.build)
403 self._download_stage0_helper(filename, pattern, tarball_suffix)
404 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
406 self._download_stage0_helper(filename, "rustc", tarball_suffix)
407 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
409 self._download_stage0_helper(filename, "cargo", tarball_suffix)
410 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
411 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
412 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
413 lib_dir = "{}/lib".format(self.bin_root())
414 for lib in os.listdir(lib_dir):
415 if lib.endswith(".so"):
416 self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
417 with output(self.rustc_stamp()) as rust_stamp:
418 rust_stamp.write(self.date)
420 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
421 not os.path.exists(self.rustfmt())
422 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
425 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
426 [channel, date] = rustfmt_channel.split('-', 1)
427 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
428 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
429 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
430 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
431 with output(self.rustfmt_stamp()) as rustfmt_stamp:
432 rustfmt_stamp.write(self.date + self.rustfmt_channel)
434 if self.downloading_llvm():
435 # We want the most recent LLVM submodule update to avoid downloading
436 # LLVM more often than necessary.
438 # This git command finds that commit SHA, looking for bors-authored
439 # merges that modified src/llvm-project.
441 # This works even in a repository that has not yet initialized
443 top_level = subprocess.check_output([
444 "git", "rev-parse", "--show-toplevel",
445 ]).decode(sys.getdefaultencoding()).strip()
446 llvm_sha = subprocess.check_output([
447 "git", "log", "--author=bors", "--format=%H", "-n1",
448 "-m", "--first-parent",
450 "{}/src/llvm-project".format(top_level),
451 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
452 ]).decode(sys.getdefaultencoding()).strip()
453 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
454 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
455 self._download_ci_llvm(llvm_sha, llvm_assertions)
456 for binary in ["llvm-config", "FileCheck"]:
457 self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
458 with output(self.llvm_stamp()) as llvm_stamp:
459 llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
461 def downloading_llvm(self):
462 opt = self.get_toml('download-ci-llvm', 'llvm')
463 return opt == "true" \
464 or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
466 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
469 cache_dst = os.path.join(self.build_dir, "cache")
470 rustc_cache = os.path.join(cache_dst, date)
471 if not os.path.exists(rustc_cache):
472 os.makedirs(rustc_cache)
474 url = "{}/dist/{}".format(self._download_url, date)
475 tarball = os.path.join(rustc_cache, filename)
476 if not os.path.exists(tarball):
477 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
478 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
480 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
481 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
482 cache_dst = os.path.join(self.build_dir, "cache")
483 rustc_cache = os.path.join(cache_dst, cache_prefix)
484 if not os.path.exists(rustc_cache):
485 os.makedirs(rustc_cache)
487 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
489 url = url.replace('rustc-builds', 'rustc-builds-alt')
490 # ci-artifacts are only stored as .xz, not .gz
492 print("error: XZ support is required to download LLVM")
493 print("help: consider disabling `download-ci-llvm` or using python3")
495 tarball_suffix = '.tar.xz'
496 filename = "rust-dev-nightly-" + self.build + tarball_suffix
497 tarball = os.path.join(rustc_cache, filename)
498 if not os.path.exists(tarball):
499 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
500 unpack(tarball, tarball_suffix, self.llvm_root(),
502 verbose=self.verbose)
504 def fix_bin_or_dylib(self, fname):
505 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
506 or the RPATH section, to fix the dynamic library search path
508 This method is only required on NixOS and uses the PatchELF utility to
509 change the interpreter/RPATH of ELF executables.
511 Please see https://nixos.org/patchelf.html for more information
513 default_encoding = sys.getdefaultencoding()
515 ostype = subprocess.check_output(
516 ['uname', '-s']).strip().decode(default_encoding)
517 except subprocess.CalledProcessError:
519 except OSError as reason:
520 if getattr(reason, 'winerror', None) is not None:
524 if ostype != "Linux":
527 if not os.path.exists("/etc/NIXOS"):
529 if os.path.exists("/lib"):
532 # At this point we're pretty sure the user is running NixOS
533 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
534 print(nix_os_msg, fname)
536 # Only build `stage0/.nix-deps` once.
537 nix_deps_dir = self.nix_deps_dir
539 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
540 if not os.path.exists(nix_deps_dir):
541 os.makedirs(nix_deps_dir)
544 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
545 "stdenv.cc.bintools",
547 # Needed as a system dependency of `libLLVM-*.so`.
550 # Needed for patching ELF binaries (see doc comment above).
554 # Run `nix-build` to "build" each dependency (which will likely reuse
555 # the existing `/nix/store` copy, or at most download a pre-built copy).
556 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
557 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
558 # ensuring garbage collection will never remove the `/nix/store` path
559 # (which would break our patched binaries that hardcode those paths).
562 subprocess.check_output([
563 "nix-build", "<nixpkgs>",
565 "-o", "{}/{}".format(nix_deps_dir, dep),
567 except subprocess.CalledProcessError as reason:
568 print("warning: failed to call nix-build:", reason)
571 self.nix_deps_dir = nix_deps_dir
573 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
575 if fname.endswith(".so"):
576 # Dynamic library, patch RPATH to point to system dependencies.
577 dylib_deps = ["zlib"]
579 # Relative default, all binary and dynamic libraries we ship
580 # appear to have this (even when `../lib` is redundant).
582 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
583 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
585 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
586 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
587 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
590 subprocess.check_output([patchelf] + patchelf_args + [fname])
591 except subprocess.CalledProcessError as reason:
592 print("warning: failed to call patchelf:", reason)
595 def rustc_stamp(self):
596 """Return the path for .rustc-stamp
599 >>> rb.build_dir = "build"
600 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
603 return os.path.join(self.bin_root(), '.rustc-stamp')
605 def rustfmt_stamp(self):
606 """Return the path for .rustfmt-stamp
609 >>> rb.build_dir = "build"
610 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
613 return os.path.join(self.bin_root(), '.rustfmt-stamp')
615 def llvm_stamp(self):
616 """Return the path for .rustfmt-stamp
619 >>> rb.build_dir = "build"
620 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
623 return os.path.join(self.llvm_root(), '.llvm-stamp')
626 def program_out_of_date(self, stamp_path, extra=""):
627 """Check if the given program stamp is out of date"""
628 if not os.path.exists(stamp_path) or self.clean:
630 with open(stamp_path, 'r') as stamp:
631 return (self.date + extra) != stamp.read()
634 """Return the binary root directory
637 >>> rb.build_dir = "build"
638 >>> rb.bin_root() == os.path.join("build", "stage0")
641 When the 'build' property is given should be a nested directory:
643 >>> rb.build = "devel"
644 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
647 return os.path.join(self.build_dir, self.build, "stage0")
650 """Return the CI LLVM root directory
653 >>> rb.build_dir = "build"
654 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
657 When the 'build' property is given should be a nested directory:
659 >>> rb.build = "devel"
660 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
663 return os.path.join(self.build_dir, self.build, "ci-llvm")
665 def get_toml(self, key, section=None):
666 """Returns the value of the given key in config.toml, otherwise returns None
669 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
670 >>> rb.get_toml("key2")
673 If the key does not exists, the result is None:
675 >>> rb.get_toml("key3") is None
678 Optionally also matches the section the key appears in
680 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
681 >>> rb.get_toml('key', 'a')
683 >>> rb.get_toml('key', 'b')
685 >>> rb.get_toml('key', 'c') is None
688 >>> rb.config_toml = 'key1 = true'
689 >>> rb.get_toml("key1")
694 for line in self.config_toml.splitlines():
695 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
696 if section_match is not None:
697 cur_section = section_match.group(1)
699 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
700 if match is not None:
701 value = match.group(1)
702 if section is None or section == cur_section:
703 return self.get_string(value) or value.strip()
707 """Return config path for cargo"""
708 return self.program_config('cargo')
711 """Return config path for rustc"""
712 return self.program_config('rustc')
715 """Return config path for rustfmt"""
716 if not self.rustfmt_channel:
718 return self.program_config('rustfmt')
720 def program_config(self, program):
721 """Return config path for the given program
724 >>> rb.config_toml = 'rustc = "rustc"\\n'
725 >>> rb.program_config('rustc')
727 >>> rb.config_toml = ''
728 >>> cargo_path = rb.program_config('cargo')
729 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
733 config = self.get_toml(program)
735 return os.path.expanduser(config)
736 return os.path.join(self.bin_root(), "bin", "{}{}".format(
737 program, self.exe_suffix()))
740 def get_string(line):
741 """Return the value between double quotes
743 >>> RustBuild.get_string(' "devel" ')
745 >>> RustBuild.get_string(" 'devel' ")
747 >>> RustBuild.get_string('devel') is None
749 >>> RustBuild.get_string(' "devel ')
752 start = line.find('"')
754 end = start + 1 + line[start + 1:].find('"')
755 return line[start + 1:end]
756 start = line.find('\'')
758 end = start + 1 + line[start + 1:].find('\'')
759 return line[start + 1:end]
764 """Return a suffix for executables"""
765 if sys.platform == 'win32':
769 def bootstrap_binary(self):
770 """Return the path of the bootstrap binary
773 >>> rb.build_dir = "build"
774 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
775 ... "debug", "bootstrap")
778 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
780 def build_bootstrap(self):
781 """Build bootstrap"""
782 build_dir = os.path.join(self.build_dir, "bootstrap")
783 if self.clean and os.path.exists(build_dir):
784 shutil.rmtree(build_dir)
785 env = os.environ.copy()
786 # `CARGO_BUILD_TARGET` breaks bootstrap build.
787 # See also: <https://github.com/rust-lang/rust/issues/70208>.
788 if "CARGO_BUILD_TARGET" in env:
789 del env["CARGO_BUILD_TARGET"]
790 env["CARGO_TARGET_DIR"] = build_dir
791 env["RUSTC"] = self.rustc()
792 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
793 (os.pathsep + env["LD_LIBRARY_PATH"]) \
794 if "LD_LIBRARY_PATH" in env else ""
795 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
796 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
797 if "DYLD_LIBRARY_PATH" in env else ""
798 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
799 (os.pathsep + env["LIBRARY_PATH"]) \
800 if "LIBRARY_PATH" in env else ""
801 # preserve existing RUSTFLAGS
802 env.setdefault("RUSTFLAGS", "")
803 env["RUSTFLAGS"] += " -Cdebuginfo=2"
805 build_section = "target.{}".format(self.build)
807 if self.get_toml("crt-static", build_section) == "true":
808 target_features += ["+crt-static"]
809 elif self.get_toml("crt-static", build_section) == "false":
810 target_features += ["-crt-static"]
812 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
813 target_linker = self.get_toml("linker", build_section)
814 if target_linker is not None:
815 env["RUSTFLAGS"] += " -C linker=" + target_linker
816 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
817 if self.get_toml("deny-warnings", "rust") != "false":
818 env["RUSTFLAGS"] += " -Dwarnings"
820 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
821 os.pathsep + env["PATH"]
822 if not os.path.isfile(self.cargo()):
823 raise Exception("no cargo executable found at `{}`".format(
825 args = [self.cargo(), "build", "--manifest-path",
826 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
827 for _ in range(1, self.verbose):
828 args.append("--verbose")
829 if self.use_locked_deps:
830 args.append("--locked")
831 if self.use_vendored_sources:
832 args.append("--frozen")
833 run(args, env=env, verbose=self.verbose)
835 def build_triple(self):
836 """Build triple as in LLVM
838 Note that `default_build_triple` is moderately expensive,
839 so use `self.build` where possible.
841 config = self.get_toml('build')
844 return default_build_triple(self.verbose)
846 def check_submodule(self, module, slow_submodules):
847 if not slow_submodules:
848 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
849 cwd=os.path.join(self.rust_root, module),
850 stdout=subprocess.PIPE)
855 def update_submodule(self, module, checked_out, recorded_submodules):
856 module_path = os.path.join(self.rust_root, module)
858 if checked_out is not None:
859 default_encoding = sys.getdefaultencoding()
860 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
861 if recorded_submodules[module] == checked_out:
864 print("Updating submodule", module)
866 run(["git", "submodule", "-q", "sync", module],
867 cwd=self.rust_root, verbose=self.verbose)
869 update_args = ["git", "submodule", "update", "--init", "--recursive"]
870 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
871 update_args.append("--progress")
872 update_args.append(module)
873 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
875 run(["git", "reset", "-q", "--hard"],
876 cwd=module_path, verbose=self.verbose)
877 run(["git", "clean", "-qdfx"],
878 cwd=module_path, verbose=self.verbose)
880 def update_submodules(self):
881 """Update submodules"""
882 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
883 self.get_toml('submodules') == "false":
886 default_encoding = sys.getdefaultencoding()
888 # check the existence and version of 'git' command
889 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
890 self.git_version = distutils.version.LooseVersion(git_version_str)
892 slow_submodules = self.get_toml('fast-submodules') == "false"
895 print('Unconditionally updating all submodules')
897 print('Updating only changed submodules')
898 default_encoding = sys.getdefaultencoding()
899 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
900 ["git", "config", "--file",
901 os.path.join(self.rust_root, ".gitmodules"),
902 "--get-regexp", "path"]
903 ).decode(default_encoding).splitlines()]
904 filtered_submodules = []
905 submodules_names = []
906 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
907 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
908 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
909 or "llvm" in self.get_toml('codegen-backends', 'rust')
910 for module in submodules:
911 if module.endswith("llvm-project"):
912 # Don't sync the llvm-project submodule if an external LLVM was
913 # provided, if we are downloading LLVM or if the LLVM backend is
914 # not being built. Also, if the submodule has been initialized
915 # already, sync it anyways so that it doesn't mess up contributor
917 if external_llvm_provided or not llvm_needed:
918 if self.get_toml('lld') != 'true' and not llvm_checked_out:
920 check = self.check_submodule(module, slow_submodules)
921 filtered_submodules.append((module, check))
922 submodules_names.append(module)
923 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
924 cwd=self.rust_root, stdout=subprocess.PIPE)
925 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
926 recorded_submodules = {}
927 for data in recorded:
929 recorded_submodules[data[3]] = data[2]
930 for module in filtered_submodules:
931 self.update_submodule(module[0], module[1], recorded_submodules)
932 print("Submodules updated in %.2f seconds" % (time() - start_time))
934 def set_normal_environment(self):
935 """Set download URL for normal environment"""
936 if 'RUSTUP_DIST_SERVER' in os.environ:
937 self._download_url = os.environ['RUSTUP_DIST_SERVER']
939 self._download_url = 'https://static.rust-lang.org'
941 def set_dev_environment(self):
942 """Set download URL for development environment"""
943 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
944 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
946 self._download_url = 'https://dev-static.rust-lang.org'
948 def check_vendored_status(self):
949 """Check that vendoring is configured properly"""
950 vendor_dir = os.path.join(self.rust_root, 'vendor')
951 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
952 if os.environ.get('USER') != os.environ['SUDO_USER']:
953 self.use_vendored_sources = True
954 print('info: looks like you are running this command under `sudo`')
955 print(' and so in order to preserve your $HOME this will now')
956 print(' use vendored sources by default.')
957 if not os.path.exists(vendor_dir):
958 print('error: vendoring required, but vendor directory does not exist.')
959 print(' Run `cargo vendor` without sudo to initialize the '
961 raise Exception("{} not found".format(vendor_dir))
963 if self.use_vendored_sources:
964 if not os.path.exists('.cargo'):
965 os.makedirs('.cargo')
966 with output('.cargo/config') as cargo_config:
968 "[source.crates-io]\n"
969 "replace-with = 'vendored-sources'\n"
970 "registry = 'https://example.com'\n"
972 "[source.vendored-sources]\n"
973 "directory = '{}/vendor'\n"
974 .format(self.rust_root))
976 if os.path.exists('.cargo'):
977 shutil.rmtree('.cargo')
979 def ensure_vendored(self):
980 """Ensure that the vendored sources are available if needed"""
981 vendor_dir = os.path.join(self.rust_root, 'vendor')
982 # Note that this does not handle updating the vendored dependencies if
983 # the rust git repository is updated. Normal development usually does
984 # not use vendoring, so hopefully this isn't too much of a problem.
985 if self.use_vendored_sources and not os.path.exists(vendor_dir):
989 "--sync=./src/tools/rust-analyzer/Cargo.toml",
990 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
991 ], verbose=self.verbose, cwd=self.rust_root)
994 def bootstrap(help_triggered):
995 """Configure, fetch, build and run the initial bootstrap"""
997 # If the user is asking for help, let them know that the whole download-and-build
998 # process has to happen before anything is printed out.
1000 print("info: Downloading and building bootstrap before processing --help")
1001 print(" command. See src/bootstrap/README.md for help with common")
1004 parser = argparse.ArgumentParser(description='Build rust')
1005 parser.add_argument('--config')
1006 parser.add_argument('--build')
1007 parser.add_argument('--clean', action='store_true')
1008 parser.add_argument('-v', '--verbose', action='count', default=0)
1010 args = [a for a in sys.argv if a != '-h' and a != '--help']
1011 args, _ = parser.parse_known_args(args)
1013 # Configure initial bootstrap
1015 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1016 build.verbose = args.verbose
1017 build.clean = args.clean
1019 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1021 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1022 if not toml_path and os.path.exists('config.toml'):
1023 toml_path = 'config.toml'
1026 if not os.path.exists(toml_path):
1027 toml_path = os.path.join(build.rust_root, toml_path)
1029 with open(toml_path) as config:
1030 build.config_toml = config.read()
1032 profile = build.get_toml('profile')
1033 if profile is not None:
1034 include_file = 'config.{}.toml'.format(profile)
1035 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1036 include_path = os.path.join(include_dir, include_file)
1037 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1038 # specific key, so appending our defaults at the end allows the user to override them
1039 with open(include_path) as included_toml:
1040 build.config_toml += os.linesep + included_toml.read()
1042 config_verbose = build.get_toml('verbose', 'build')
1043 if config_verbose is not None:
1044 build.verbose = max(build.verbose, int(config_verbose))
1046 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1048 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1050 build.check_vendored_status()
1052 build_dir = build.get_toml('build-dir', 'build') or 'build'
1053 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1055 data = stage0_data(build.rust_root)
1056 build.date = data['date']
1057 build.rustc_channel = data['rustc']
1059 if "rustfmt" in data:
1060 build.rustfmt_channel = data['rustfmt']
1063 build.set_dev_environment()
1065 build.set_normal_environment()
1067 build.update_submodules()
1069 # Fetch/build the bootstrap
1070 build.build = args.build or build.build_triple()
1071 build.download_stage0()
1073 build.ensure_vendored()
1074 build.build_bootstrap()
1078 args = [build.bootstrap_binary()]
1079 args.extend(sys.argv[1:])
1080 env = os.environ.copy()
1081 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1082 env["BOOTSTRAP_PYTHON"] = sys.executable
1083 env["BUILD_DIR"] = build.build_dir
1084 env["RUSTC_BOOTSTRAP"] = '1'
1086 env["BOOTSTRAP_CONFIG"] = toml_path
1087 run(args, env=env, verbose=build.verbose)
1091 """Entry point for the bootstrap process"""
1094 # x.py help <cmd> ...
1095 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1096 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1099 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1101 bootstrap(help_triggered)
1102 if not help_triggered:
1103 print("Build completed successfully in {}".format(
1104 format_build_time(time() - start_time)))
1105 except (SystemExit, KeyboardInterrupt) as error:
1106 if hasattr(error, 'code') and isinstance(error.code, int):
1107 exit_code = error.code
1111 if not help_triggered:
1112 print("Build completed unsuccessfully in {}".format(
1113 format_build_time(time() - start_time)))
1117 if __name__ == '__main__':