1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"])
198 version = version.decode(default_encoding)
199 host = next(x for x in version.split('\n') if x.startswith("host: "))
200 triple = host.split("host: ")[1]
202 print("detected default triple {}".format(triple))
204 except Exception as e:
206 print("rustup not detected: {}".format(e))
207 print("falling back to auto-detect")
209 required = sys.platform != 'win32'
210 ostype = require(["uname", "-s"], exit=required)
211 cputype = require(['uname', '-m'], exit=required)
213 # If we do not have `uname`, assume Windows.
214 if ostype is None or cputype is None:
215 return 'x86_64-pc-windows-msvc'
217 ostype = ostype.decode(default_encoding)
218 cputype = cputype.decode(default_encoding)
220 # The goal here is to come up with the same triple as LLVM would,
221 # at least for the subset of platforms we're willing to target.
223 'Darwin': 'apple-darwin',
224 'DragonFly': 'unknown-dragonfly',
225 'FreeBSD': 'unknown-freebsd',
226 'Haiku': 'unknown-haiku',
227 'NetBSD': 'unknown-netbsd',
228 'OpenBSD': 'unknown-openbsd'
231 # Consider the direct transformation first and then the special cases
232 if ostype in ostype_mapper:
233 ostype = ostype_mapper[ostype]
234 elif ostype == 'Linux':
235 os_from_sp = subprocess.check_output(
236 ['uname', '-o']).strip().decode(default_encoding)
237 if os_from_sp == 'Android':
238 ostype = 'linux-android'
240 ostype = 'unknown-linux-gnu'
241 elif ostype == 'SunOS':
242 ostype = 'sun-solaris'
243 # On Solaris, uname -m will return a machine classification instead
244 # of a cpu type, so uname -p is recommended instead. However, the
245 # output from that option is too generic for our purposes (it will
246 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
247 # must be used instead.
248 cputype = require(['isainfo', '-k']).decode(default_encoding)
249 elif ostype.startswith('MINGW'):
250 # msys' `uname` does not print gcc configuration, but prints msys
251 # configuration. so we cannot believe `uname -m`:
252 # msys1 is always i686 and msys2 is always x86_64.
253 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
255 ostype = 'pc-windows-gnu'
257 if os.environ.get('MSYSTEM') == 'MINGW64':
259 elif ostype.startswith('MSYS'):
260 ostype = 'pc-windows-gnu'
261 elif ostype.startswith('CYGWIN_NT'):
263 if ostype.endswith('WOW64'):
265 ostype = 'pc-windows-gnu'
266 elif sys.platform == 'win32':
267 # Some Windows platforms might have a `uname` command that returns a
268 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
269 # these cases, fall back to using sys.platform.
270 return 'x86_64-pc-windows-msvc'
272 err = "unknown OS type: {}".format(ostype)
275 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
276 cputype = subprocess.check_output(
277 ['uname', '-p']).strip().decode(default_encoding)
280 'aarch64': 'aarch64',
287 'powerpc': 'powerpc',
288 'powerpc64': 'powerpc64',
289 'powerpc64le': 'powerpc64le',
291 'ppc64': 'powerpc64',
292 'ppc64le': 'powerpc64le',
300 # Consider the direct transformation first and then the special cases
301 if cputype in cputype_mapper:
302 cputype = cputype_mapper[cputype]
303 elif cputype in {'xscale', 'arm'}:
305 if ostype == 'linux-android':
306 ostype = 'linux-androideabi'
307 elif ostype == 'unknown-freebsd':
308 cputype = subprocess.check_output(
309 ['uname', '-p']).strip().decode(default_encoding)
310 ostype = 'unknown-freebsd'
311 elif cputype == 'armv6l':
313 if ostype == 'linux-android':
314 ostype = 'linux-androideabi'
317 elif cputype in {'armv7l', 'armv8l'}:
319 if ostype == 'linux-android':
320 ostype = 'linux-androideabi'
323 elif cputype == 'mips':
324 if sys.byteorder == 'big':
326 elif sys.byteorder == 'little':
329 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
330 elif cputype == 'mips64':
331 if sys.byteorder == 'big':
333 elif sys.byteorder == 'little':
336 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
337 # only the n64 ABI is supported, indicate it
339 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
342 err = "unknown cpu type: {}".format(cputype)
345 return "{}-{}".format(cputype, ostype)
348 @contextlib.contextmanager
349 def output(filepath):
350 tmp = filepath + '.tmp'
351 with open(tmp, 'w') as f:
354 if os.path.exists(filepath):
355 os.remove(filepath) # PermissionError/OSError on Win32 if in use
357 shutil.copy2(tmp, filepath)
360 os.rename(tmp, filepath)
363 class RustBuild(object):
364 """Provide all the methods required to build Rust"""
367 self._download_url = ''
368 self.rustc_channel = ''
369 self.rustfmt_channel = ''
373 self.config_toml = ''
375 self.use_locked_deps = ''
376 self.use_vendored_sources = ''
378 self.git_version = None
379 self.nix_deps_dir = None
381 def download_stage0(self):
382 """Fetch the build system for Rust, written in Rust
384 This method will build a cache directory, then it will fetch the
385 tarball which has the stage0 compiler used to then bootstrap the Rust
388 Each downloaded tarball is extracted, after that, the script
389 will move all the content to the right place.
391 rustc_channel = self.rustc_channel
392 rustfmt_channel = self.rustfmt_channel
394 if self.rustc().startswith(self.bin_root()) and \
395 (not os.path.exists(self.rustc()) or
396 self.program_out_of_date(self.rustc_stamp())):
397 if os.path.exists(self.bin_root()):
398 shutil.rmtree(self.bin_root())
399 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
400 filename = "rust-std-{}-{}{}".format(
401 rustc_channel, self.build, tarball_suffix)
402 pattern = "rust-std-{}".format(self.build)
403 self._download_stage0_helper(filename, pattern, tarball_suffix)
404 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
406 self._download_stage0_helper(filename, "rustc", tarball_suffix)
407 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
409 self._download_stage0_helper(filename, "cargo", tarball_suffix)
410 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
411 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
412 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
413 lib_dir = "{}/lib".format(self.bin_root())
414 for lib in os.listdir(lib_dir):
415 if lib.endswith(".so"):
416 self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
417 with output(self.rustc_stamp()) as rust_stamp:
418 rust_stamp.write(self.date)
420 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
421 not os.path.exists(self.rustfmt())
422 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
425 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
426 [channel, date] = rustfmt_channel.split('-', 1)
427 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
428 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
429 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
430 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
431 with output(self.rustfmt_stamp()) as rustfmt_stamp:
432 rustfmt_stamp.write(self.date + self.rustfmt_channel)
434 if self.downloading_llvm():
435 # We want the most recent LLVM submodule update to avoid downloading
436 # LLVM more often than necessary.
438 # This git command finds that commit SHA, looking for bors-authored
439 # merges that modified src/llvm-project.
441 # This works even in a repository that has not yet initialized
443 top_level = subprocess.check_output([
444 "git", "rev-parse", "--show-toplevel",
445 ]).decode(sys.getdefaultencoding()).strip()
446 llvm_sha = subprocess.check_output([
447 "git", "log", "--author=bors", "--format=%H", "-n1",
448 "-m", "--first-parent",
450 "{}/src/llvm-project".format(top_level),
451 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
452 ]).decode(sys.getdefaultencoding()).strip()
453 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
454 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
455 self._download_ci_llvm(llvm_sha, llvm_assertions)
456 for binary in ["llvm-config", "FileCheck"]:
457 self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
458 with output(self.llvm_stamp()) as llvm_stamp:
459 llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
461 def downloading_llvm(self):
462 opt = self.get_toml('download-ci-llvm', 'llvm')
463 return opt == "true" \
464 or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
466 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
469 cache_dst = os.path.join(self.build_dir, "cache")
470 rustc_cache = os.path.join(cache_dst, date)
471 if not os.path.exists(rustc_cache):
472 os.makedirs(rustc_cache)
474 url = "{}/dist/{}".format(self._download_url, date)
475 tarball = os.path.join(rustc_cache, filename)
476 if not os.path.exists(tarball):
477 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
478 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
480 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
481 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
482 cache_dst = os.path.join(self.build_dir, "cache")
483 rustc_cache = os.path.join(cache_dst, cache_prefix)
484 if not os.path.exists(rustc_cache):
485 os.makedirs(rustc_cache)
487 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
489 url = url.replace('rustc-builds', 'rustc-builds-alt')
490 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
491 filename = "rust-dev-nightly-" + self.build + tarball_suffix
492 tarball = os.path.join(rustc_cache, filename)
493 if not os.path.exists(tarball):
494 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
495 unpack(tarball, tarball_suffix, self.llvm_root(),
497 verbose=self.verbose)
499 def fix_bin_or_dylib(self, fname):
500 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
501 or the RPATH section, to fix the dynamic library search path
503 This method is only required on NixOS and uses the PatchELF utility to
504 change the interpreter/RPATH of ELF executables.
506 Please see https://nixos.org/patchelf.html for more information
508 default_encoding = sys.getdefaultencoding()
510 ostype = subprocess.check_output(
511 ['uname', '-s']).strip().decode(default_encoding)
512 except subprocess.CalledProcessError:
514 except OSError as reason:
515 if getattr(reason, 'winerror', None) is not None:
519 if ostype != "Linux":
522 if not os.path.exists("/etc/NIXOS"):
524 if os.path.exists("/lib"):
527 # At this point we're pretty sure the user is running NixOS
528 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
529 print(nix_os_msg, fname)
531 # Only build `stage0/.nix-deps` once.
532 nix_deps_dir = self.nix_deps_dir
534 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
535 if not os.path.exists(nix_deps_dir):
536 os.makedirs(nix_deps_dir)
539 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
540 "stdenv.cc.bintools",
542 # Needed as a system dependency of `libLLVM-*.so`.
545 # Needed for patching ELF binaries (see doc comment above).
549 # Run `nix-build` to "build" each dependency (which will likely reuse
550 # the existing `/nix/store` copy, or at most download a pre-built copy).
551 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
552 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
553 # ensuring garbage collection will never remove the `/nix/store` path
554 # (which would break our patched binaries that hardcode those paths).
557 subprocess.check_output([
558 "nix-build", "<nixpkgs>",
560 "-o", "{}/{}".format(nix_deps_dir, dep),
562 except subprocess.CalledProcessError as reason:
563 print("warning: failed to call nix-build:", reason)
566 self.nix_deps_dir = nix_deps_dir
568 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
570 if fname.endswith(".so"):
571 # Dynamic library, patch RPATH to point to system dependencies.
572 dylib_deps = ["zlib"]
574 # Relative default, all binary and dynamic libraries we ship
575 # appear to have this (even when `../lib` is redundant).
577 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
578 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
580 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
581 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
582 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
585 subprocess.check_output([patchelf] + patchelf_args + [fname])
586 except subprocess.CalledProcessError as reason:
587 print("warning: failed to call patchelf:", reason)
590 def rustc_stamp(self):
591 """Return the path for .rustc-stamp
594 >>> rb.build_dir = "build"
595 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
598 return os.path.join(self.bin_root(), '.rustc-stamp')
600 def rustfmt_stamp(self):
601 """Return the path for .rustfmt-stamp
604 >>> rb.build_dir = "build"
605 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
608 return os.path.join(self.bin_root(), '.rustfmt-stamp')
610 def llvm_stamp(self):
611 """Return the path for .rustfmt-stamp
614 >>> rb.build_dir = "build"
615 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
618 return os.path.join(self.llvm_root(), '.llvm-stamp')
621 def program_out_of_date(self, stamp_path, extra=""):
622 """Check if the given program stamp is out of date"""
623 if not os.path.exists(stamp_path) or self.clean:
625 with open(stamp_path, 'r') as stamp:
626 return (self.date + extra) != stamp.read()
629 """Return the binary root directory
632 >>> rb.build_dir = "build"
633 >>> rb.bin_root() == os.path.join("build", "stage0")
636 When the 'build' property is given should be a nested directory:
638 >>> rb.build = "devel"
639 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
642 return os.path.join(self.build_dir, self.build, "stage0")
645 """Return the CI LLVM root directory
648 >>> rb.build_dir = "build"
649 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
652 When the 'build' property is given should be a nested directory:
654 >>> rb.build = "devel"
655 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
658 return os.path.join(self.build_dir, self.build, "ci-llvm")
660 def get_toml(self, key, section=None):
661 """Returns the value of the given key in config.toml, otherwise returns None
664 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
665 >>> rb.get_toml("key2")
668 If the key does not exists, the result is None:
670 >>> rb.get_toml("key3") is None
673 Optionally also matches the section the key appears in
675 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
676 >>> rb.get_toml('key', 'a')
678 >>> rb.get_toml('key', 'b')
680 >>> rb.get_toml('key', 'c') is None
683 >>> rb.config_toml = 'key1 = true'
684 >>> rb.get_toml("key1")
689 for line in self.config_toml.splitlines():
690 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
691 if section_match is not None:
692 cur_section = section_match.group(1)
694 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
695 if match is not None:
696 value = match.group(1)
697 if section is None or section == cur_section:
698 return self.get_string(value) or value.strip()
702 """Return config path for cargo"""
703 return self.program_config('cargo')
706 """Return config path for rustc"""
707 return self.program_config('rustc')
710 """Return config path for rustfmt"""
711 if not self.rustfmt_channel:
713 return self.program_config('rustfmt')
715 def program_config(self, program):
716 """Return config path for the given program
719 >>> rb.config_toml = 'rustc = "rustc"\\n'
720 >>> rb.program_config('rustc')
722 >>> rb.config_toml = ''
723 >>> cargo_path = rb.program_config('cargo')
724 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
728 config = self.get_toml(program)
730 return os.path.expanduser(config)
731 return os.path.join(self.bin_root(), "bin", "{}{}".format(
732 program, self.exe_suffix()))
735 def get_string(line):
736 """Return the value between double quotes
738 >>> RustBuild.get_string(' "devel" ')
740 >>> RustBuild.get_string(" 'devel' ")
742 >>> RustBuild.get_string('devel') is None
744 >>> RustBuild.get_string(' "devel ')
747 start = line.find('"')
749 end = start + 1 + line[start + 1:].find('"')
750 return line[start + 1:end]
751 start = line.find('\'')
753 end = start + 1 + line[start + 1:].find('\'')
754 return line[start + 1:end]
759 """Return a suffix for executables"""
760 if sys.platform == 'win32':
764 def bootstrap_binary(self):
765 """Return the path of the bootstrap binary
768 >>> rb.build_dir = "build"
769 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
770 ... "debug", "bootstrap")
773 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
775 def build_bootstrap(self):
776 """Build bootstrap"""
777 build_dir = os.path.join(self.build_dir, "bootstrap")
778 if self.clean and os.path.exists(build_dir):
779 shutil.rmtree(build_dir)
780 env = os.environ.copy()
781 # `CARGO_BUILD_TARGET` breaks bootstrap build.
782 # See also: <https://github.com/rust-lang/rust/issues/70208>.
783 if "CARGO_BUILD_TARGET" in env:
784 del env["CARGO_BUILD_TARGET"]
785 env["CARGO_TARGET_DIR"] = build_dir
786 env["RUSTC"] = self.rustc()
787 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
788 (os.pathsep + env["LD_LIBRARY_PATH"]) \
789 if "LD_LIBRARY_PATH" in env else ""
790 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
791 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
792 if "DYLD_LIBRARY_PATH" in env else ""
793 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
794 (os.pathsep + env["LIBRARY_PATH"]) \
795 if "LIBRARY_PATH" in env else ""
796 # preserve existing RUSTFLAGS
797 env.setdefault("RUSTFLAGS", "")
798 env["RUSTFLAGS"] += " -Cdebuginfo=2"
800 build_section = "target.{}".format(self.build)
802 if self.get_toml("crt-static", build_section) == "true":
803 target_features += ["+crt-static"]
804 elif self.get_toml("crt-static", build_section) == "false":
805 target_features += ["-crt-static"]
807 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
808 target_linker = self.get_toml("linker", build_section)
809 if target_linker is not None:
810 env["RUSTFLAGS"] += " -C linker=" + target_linker
811 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
812 if self.get_toml("deny-warnings", "rust") != "false":
813 env["RUSTFLAGS"] += " -Dwarnings"
815 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
816 os.pathsep + env["PATH"]
817 if not os.path.isfile(self.cargo()):
818 raise Exception("no cargo executable found at `{}`".format(
820 args = [self.cargo(), "build", "--manifest-path",
821 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
822 for _ in range(1, self.verbose):
823 args.append("--verbose")
824 if self.use_locked_deps:
825 args.append("--locked")
826 if self.use_vendored_sources:
827 args.append("--frozen")
828 run(args, env=env, verbose=self.verbose)
830 def build_triple(self):
831 """Build triple as in LLVM
833 Note that `default_build_triple` is moderately expensive,
834 so use `self.build` where possible.
836 config = self.get_toml('build')
839 return default_build_triple(self.verbose)
841 def check_submodule(self, module, slow_submodules):
842 if not slow_submodules:
843 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
844 cwd=os.path.join(self.rust_root, module),
845 stdout=subprocess.PIPE)
850 def update_submodule(self, module, checked_out, recorded_submodules):
851 module_path = os.path.join(self.rust_root, module)
853 if checked_out is not None:
854 default_encoding = sys.getdefaultencoding()
855 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
856 if recorded_submodules[module] == checked_out:
859 print("Updating submodule", module)
861 run(["git", "submodule", "-q", "sync", module],
862 cwd=self.rust_root, verbose=self.verbose)
864 update_args = ["git", "submodule", "update", "--init", "--recursive"]
865 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
866 update_args.append("--progress")
867 update_args.append(module)
868 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
870 run(["git", "reset", "-q", "--hard"],
871 cwd=module_path, verbose=self.verbose)
872 run(["git", "clean", "-qdfx"],
873 cwd=module_path, verbose=self.verbose)
875 def update_submodules(self):
876 """Update submodules"""
877 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
878 self.get_toml('submodules') == "false":
881 default_encoding = sys.getdefaultencoding()
883 # check the existence and version of 'git' command
884 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
885 self.git_version = distutils.version.LooseVersion(git_version_str)
887 slow_submodules = self.get_toml('fast-submodules') == "false"
890 print('Unconditionally updating all submodules')
892 print('Updating only changed submodules')
893 default_encoding = sys.getdefaultencoding()
894 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
895 ["git", "config", "--file",
896 os.path.join(self.rust_root, ".gitmodules"),
897 "--get-regexp", "path"]
898 ).decode(default_encoding).splitlines()]
899 filtered_submodules = []
900 submodules_names = []
901 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
902 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
903 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
904 or "llvm" in self.get_toml('codegen-backends', 'rust')
905 for module in submodules:
906 if module.endswith("llvm-project"):
907 # Don't sync the llvm-project submodule if an external LLVM was
908 # provided, if we are downloading LLVM or if the LLVM backend is
909 # not being built. Also, if the submodule has been initialized
910 # already, sync it anyways so that it doesn't mess up contributor
912 if external_llvm_provided or not llvm_needed:
913 if self.get_toml('lld') != 'true' and not llvm_checked_out:
915 check = self.check_submodule(module, slow_submodules)
916 filtered_submodules.append((module, check))
917 submodules_names.append(module)
918 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
919 cwd=self.rust_root, stdout=subprocess.PIPE)
920 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
921 recorded_submodules = {}
922 for data in recorded:
924 recorded_submodules[data[3]] = data[2]
925 for module in filtered_submodules:
926 self.update_submodule(module[0], module[1], recorded_submodules)
927 print("Submodules updated in %.2f seconds" % (time() - start_time))
929 def set_normal_environment(self):
930 """Set download URL for normal environment"""
931 if 'RUSTUP_DIST_SERVER' in os.environ:
932 self._download_url = os.environ['RUSTUP_DIST_SERVER']
934 self._download_url = 'https://static.rust-lang.org'
936 def set_dev_environment(self):
937 """Set download URL for development environment"""
938 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
939 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
941 self._download_url = 'https://dev-static.rust-lang.org'
943 def check_vendored_status(self):
944 """Check that vendoring is configured properly"""
945 vendor_dir = os.path.join(self.rust_root, 'vendor')
946 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
947 if os.environ.get('USER') != os.environ['SUDO_USER']:
948 self.use_vendored_sources = True
949 print('info: looks like you are running this command under `sudo`')
950 print(' and so in order to preserve your $HOME this will now')
951 print(' use vendored sources by default.')
952 if not os.path.exists(vendor_dir):
953 print('error: vendoring required, but vendor directory does not exist.')
954 print(' Run `cargo vendor` without sudo to initialize the '
956 raise Exception("{} not found".format(vendor_dir))
958 if self.use_vendored_sources:
959 if not os.path.exists('.cargo'):
960 os.makedirs('.cargo')
961 with output('.cargo/config') as cargo_config:
963 "[source.crates-io]\n"
964 "replace-with = 'vendored-sources'\n"
965 "registry = 'https://example.com'\n"
967 "[source.vendored-sources]\n"
968 "directory = '{}/vendor'\n"
969 .format(self.rust_root))
971 if os.path.exists('.cargo'):
972 shutil.rmtree('.cargo')
974 def ensure_vendored(self):
975 """Ensure that the vendored sources are available if needed"""
976 vendor_dir = os.path.join(self.rust_root, 'vendor')
977 # Note that this does not handle updating the vendored dependencies if
978 # the rust git repository is updated. Normal development usually does
979 # not use vendoring, so hopefully this isn't too much of a problem.
980 if self.use_vendored_sources and not os.path.exists(vendor_dir):
984 "--sync=./src/tools/rust-analyzer/Cargo.toml",
985 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
986 ], verbose=self.verbose, cwd=self.rust_root)
989 def bootstrap(help_triggered):
990 """Configure, fetch, build and run the initial bootstrap"""
992 # If the user is asking for help, let them know that the whole download-and-build
993 # process has to happen before anything is printed out.
995 print("info: Downloading and building bootstrap before processing --help")
996 print(" command. See src/bootstrap/README.md for help with common")
999 parser = argparse.ArgumentParser(description='Build rust')
1000 parser.add_argument('--config')
1001 parser.add_argument('--build')
1002 parser.add_argument('--clean', action='store_true')
1003 parser.add_argument('-v', '--verbose', action='count', default=0)
1005 args = [a for a in sys.argv if a != '-h' and a != '--help']
1006 args, _ = parser.parse_known_args(args)
1008 # Configure initial bootstrap
1010 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1011 build.verbose = args.verbose
1012 build.clean = args.clean
1014 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1016 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1017 if not toml_path and os.path.exists('config.toml'):
1018 toml_path = 'config.toml'
1021 if not os.path.exists(toml_path):
1022 toml_path = os.path.join(build.rust_root, toml_path)
1024 with open(toml_path) as config:
1025 build.config_toml = config.read()
1027 profile = build.get_toml('profile')
1028 if profile is not None:
1029 include_file = 'config.{}.toml'.format(profile)
1030 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1031 include_path = os.path.join(include_dir, include_file)
1032 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1033 # specific key, so appending our defaults at the end allows the user to override them
1034 with open(include_path) as included_toml:
1035 build.config_toml += os.linesep + included_toml.read()
1037 config_verbose = build.get_toml('verbose', 'build')
1038 if config_verbose is not None:
1039 build.verbose = max(build.verbose, int(config_verbose))
1041 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1043 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1045 build.check_vendored_status()
1047 build_dir = build.get_toml('build-dir', 'build') or 'build'
1048 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1050 data = stage0_data(build.rust_root)
1051 build.date = data['date']
1052 build.rustc_channel = data['rustc']
1054 if "rustfmt" in data:
1055 build.rustfmt_channel = data['rustfmt']
1058 build.set_dev_environment()
1060 build.set_normal_environment()
1062 build.update_submodules()
1064 # Fetch/build the bootstrap
1065 build.build = args.build or build.build_triple()
1066 build.download_stage0()
1068 build.ensure_vendored()
1069 build.build_bootstrap()
1073 args = [build.bootstrap_binary()]
1074 args.extend(sys.argv[1:])
1075 env = os.environ.copy()
1076 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1077 env["BOOTSTRAP_PYTHON"] = sys.executable
1078 env["BUILD_DIR"] = build.build_dir
1079 env["RUSTC_BOOTSTRAP"] = '1'
1081 env["BOOTSTRAP_CONFIG"] = toml_path
1082 run(args, env=env, verbose=build.verbose)
1086 """Entry point for the bootstrap process"""
1089 # x.py help <cmd> ...
1090 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1091 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1094 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1096 bootstrap(help_triggered)
1097 if not help_triggered:
1098 print("Build completed successfully in {}".format(
1099 format_build_time(time() - start_time)))
1100 except (SystemExit, KeyboardInterrupt) as error:
1101 if hasattr(error, 'code') and isinstance(error.code, int):
1102 exit_code = error.code
1106 if not help_triggered:
1107 print("Build completed unsuccessfully in {}".format(
1108 format_build_time(time() - start_time)))
1112 if __name__ == '__main__':