1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple(verbose):
191 """Build triple as in LLVM"""
192 # If the user already has a host build triple with an existing `rustc`
193 # install, use their preference. This fixes most issues with Windows builds
194 # being detected as GNU instead of MSVC.
195 default_encoding = sys.getdefaultencoding()
197 version = subprocess.check_output(["rustc", "--version", "--verbose"])
198 version = version.decode(default_encoding)
199 host = next(x for x in version.split('\n') if x.startswith("host: "))
200 triple = host.split("host: ")[1]
202 print("detected default triple {}".format(triple))
204 except Exception as e:
206 print("rustup not detected: {}".format(e))
207 print("falling back to auto-detect")
209 required = sys.platform != 'win32'
210 ostype = require(["uname", "-s"], exit=required)
211 cputype = require(['uname', '-m'], exit=required)
213 # If we do not have `uname`, assume Windows.
214 if ostype is None or cputype is None:
215 return 'x86_64-pc-windows-msvc'
217 ostype = ostype.decode(default_encoding)
218 cputype = cputype.decode(default_encoding)
220 # The goal here is to come up with the same triple as LLVM would,
221 # at least for the subset of platforms we're willing to target.
223 'Darwin': 'apple-darwin',
224 'DragonFly': 'unknown-dragonfly',
225 'FreeBSD': 'unknown-freebsd',
226 'Haiku': 'unknown-haiku',
227 'NetBSD': 'unknown-netbsd',
228 'OpenBSD': 'unknown-openbsd'
231 # Consider the direct transformation first and then the special cases
232 if ostype in ostype_mapper:
233 ostype = ostype_mapper[ostype]
234 elif ostype == 'Linux':
235 os_from_sp = subprocess.check_output(
236 ['uname', '-o']).strip().decode(default_encoding)
237 if os_from_sp == 'Android':
238 ostype = 'linux-android'
240 ostype = 'unknown-linux-gnu'
241 elif ostype == 'SunOS':
242 ostype = 'sun-solaris'
243 # On Solaris, uname -m will return a machine classification instead
244 # of a cpu type, so uname -p is recommended instead. However, the
245 # output from that option is too generic for our purposes (it will
246 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
247 # must be used instead.
248 cputype = require(['isainfo', '-k']).decode(default_encoding)
249 elif ostype.startswith('MINGW'):
250 # msys' `uname` does not print gcc configuration, but prints msys
251 # configuration. so we cannot believe `uname -m`:
252 # msys1 is always i686 and msys2 is always x86_64.
253 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
255 ostype = 'pc-windows-gnu'
257 if os.environ.get('MSYSTEM') == 'MINGW64':
259 elif ostype.startswith('MSYS'):
260 ostype = 'pc-windows-gnu'
261 elif ostype.startswith('CYGWIN_NT'):
263 if ostype.endswith('WOW64'):
265 ostype = 'pc-windows-gnu'
266 elif sys.platform == 'win32':
267 # Some Windows platforms might have a `uname` command that returns a
268 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
269 # these cases, fall back to using sys.platform.
270 return 'x86_64-pc-windows-msvc'
272 err = "unknown OS type: {}".format(ostype)
275 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
276 cputype = subprocess.check_output(
277 ['uname', '-p']).strip().decode(default_encoding)
280 'aarch64': 'aarch64',
287 'powerpc': 'powerpc',
288 'powerpc64': 'powerpc64',
289 'powerpc64le': 'powerpc64le',
291 'ppc64': 'powerpc64',
292 'ppc64le': 'powerpc64le',
300 # Consider the direct transformation first and then the special cases
301 if cputype in cputype_mapper:
302 cputype = cputype_mapper[cputype]
303 elif cputype in {'xscale', 'arm'}:
305 if ostype == 'linux-android':
306 ostype = 'linux-androideabi'
307 elif ostype == 'unknown-freebsd':
308 cputype = subprocess.check_output(
309 ['uname', '-p']).strip().decode(default_encoding)
310 ostype = 'unknown-freebsd'
311 elif cputype == 'armv6l':
313 if ostype == 'linux-android':
314 ostype = 'linux-androideabi'
317 elif cputype in {'armv7l', 'armv8l'}:
319 if ostype == 'linux-android':
320 ostype = 'linux-androideabi'
323 elif cputype == 'mips':
324 if sys.byteorder == 'big':
326 elif sys.byteorder == 'little':
329 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
330 elif cputype == 'mips64':
331 if sys.byteorder == 'big':
333 elif sys.byteorder == 'little':
336 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
337 # only the n64 ABI is supported, indicate it
339 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
342 err = "unknown cpu type: {}".format(cputype)
345 return "{}-{}".format(cputype, ostype)
348 @contextlib.contextmanager
349 def output(filepath):
350 tmp = filepath + '.tmp'
351 with open(tmp, 'w') as f:
354 os.remove(filepath) # PermissionError/OSError on Win32 if in use
355 os.rename(tmp, filepath)
357 shutil.copy2(tmp, filepath)
361 class RustBuild(object):
362 """Provide all the methods required to build Rust"""
365 self._download_url = ''
366 self.rustc_channel = ''
367 self.rustfmt_channel = ''
371 self.config_toml = ''
373 self.use_locked_deps = ''
374 self.use_vendored_sources = ''
376 self.git_version = None
377 self.nix_deps_dir = None
379 def download_stage0(self):
380 """Fetch the build system for Rust, written in Rust
382 This method will build a cache directory, then it will fetch the
383 tarball which has the stage0 compiler used to then bootstrap the Rust
386 Each downloaded tarball is extracted, after that, the script
387 will move all the content to the right place.
389 rustc_channel = self.rustc_channel
390 rustfmt_channel = self.rustfmt_channel
392 if self.rustc().startswith(self.bin_root()) and \
393 (not os.path.exists(self.rustc()) or
394 self.program_out_of_date(self.rustc_stamp())):
395 if os.path.exists(self.bin_root()):
396 shutil.rmtree(self.bin_root())
397 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
398 filename = "rust-std-{}-{}{}".format(
399 rustc_channel, self.build, tarball_suffix)
400 pattern = "rust-std-{}".format(self.build)
401 self._download_stage0_helper(filename, pattern, tarball_suffix)
402 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
404 self._download_stage0_helper(filename, "rustc", tarball_suffix)
405 filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
407 self._download_stage0_helper(filename, "cargo", tarball_suffix)
408 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
409 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
410 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
411 lib_dir = "{}/lib".format(self.bin_root())
412 for lib in os.listdir(lib_dir):
413 if lib.endswith(".so"):
414 self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
415 with output(self.rustc_stamp()) as rust_stamp:
416 rust_stamp.write(self.date)
418 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
419 not os.path.exists(self.rustfmt())
420 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
423 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
424 [channel, date] = rustfmt_channel.split('-', 1)
425 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
426 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
427 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
428 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
429 with output(self.rustfmt_stamp()) as rustfmt_stamp:
430 rustfmt_stamp.write(self.date + self.rustfmt_channel)
432 if self.downloading_llvm():
433 # We want the most recent LLVM submodule update to avoid downloading
434 # LLVM more often than necessary.
436 # This git command finds that commit SHA, looking for bors-authored
437 # merges that modified src/llvm-project.
439 # This works even in a repository that has not yet initialized
441 top_level = subprocess.check_output([
442 "git", "rev-parse", "--show-toplevel",
443 ]).decode(sys.getdefaultencoding()).strip()
444 llvm_sha = subprocess.check_output([
445 "git", "log", "--author=bors", "--format=%H", "-n1",
446 "-m", "--first-parent",
448 "{}/src/llvm-project".format(top_level),
449 "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
450 ]).decode(sys.getdefaultencoding()).strip()
451 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
452 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
453 self._download_ci_llvm(llvm_sha, llvm_assertions)
454 for binary in ["llvm-config", "FileCheck"]:
455 self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
456 with output(self.llvm_stamp()) as llvm_stamp:
457 llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
459 def downloading_llvm(self):
460 opt = self.get_toml('download-ci-llvm', 'llvm')
461 return opt == "true" \
462 or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
464 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
467 cache_dst = os.path.join(self.build_dir, "cache")
468 rustc_cache = os.path.join(cache_dst, date)
469 if not os.path.exists(rustc_cache):
470 os.makedirs(rustc_cache)
472 url = "{}/dist/{}".format(self._download_url, date)
473 tarball = os.path.join(rustc_cache, filename)
474 if not os.path.exists(tarball):
475 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
476 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
478 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
479 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
480 cache_dst = os.path.join(self.build_dir, "cache")
481 rustc_cache = os.path.join(cache_dst, cache_prefix)
482 if not os.path.exists(rustc_cache):
483 os.makedirs(rustc_cache)
485 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
487 url = url.replace('rustc-builds', 'rustc-builds-alt')
488 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
489 filename = "rust-dev-nightly-" + self.build + tarball_suffix
490 tarball = os.path.join(rustc_cache, filename)
491 if not os.path.exists(tarball):
492 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
493 unpack(tarball, tarball_suffix, self.llvm_root(),
495 verbose=self.verbose)
497 def fix_bin_or_dylib(self, fname):
498 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
499 or the RPATH section, to fix the dynamic library search path
501 This method is only required on NixOS and uses the PatchELF utility to
502 change the interpreter/RPATH of ELF executables.
504 Please see https://nixos.org/patchelf.html for more information
506 default_encoding = sys.getdefaultencoding()
508 ostype = subprocess.check_output(
509 ['uname', '-s']).strip().decode(default_encoding)
510 except subprocess.CalledProcessError:
512 except OSError as reason:
513 if getattr(reason, 'winerror', None) is not None:
517 if ostype != "Linux":
520 if not os.path.exists("/etc/NIXOS"):
522 if os.path.exists("/lib"):
525 # At this point we're pretty sure the user is running NixOS
526 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
527 print(nix_os_msg, fname)
529 # Only build `stage0/.nix-deps` once.
530 nix_deps_dir = self.nix_deps_dir
532 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
533 if not os.path.exists(nix_deps_dir):
534 os.makedirs(nix_deps_dir)
537 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
538 "stdenv.cc.bintools",
540 # Needed as a system dependency of `libLLVM-*.so`.
543 # Needed for patching ELF binaries (see doc comment above).
547 # Run `nix-build` to "build" each dependency (which will likely reuse
548 # the existing `/nix/store` copy, or at most download a pre-built copy).
549 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
550 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
551 # ensuring garbage collection will never remove the `/nix/store` path
552 # (which would break our patched binaries that hardcode those paths).
555 subprocess.check_output([
556 "nix-build", "<nixpkgs>",
558 "-o", "{}/{}".format(nix_deps_dir, dep),
560 except subprocess.CalledProcessError as reason:
561 print("warning: failed to call nix-build:", reason)
564 self.nix_deps_dir = nix_deps_dir
566 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
568 if fname.endswith(".so"):
569 # Dynamic library, patch RPATH to point to system dependencies.
570 dylib_deps = ["zlib"]
572 # Relative default, all binary and dynamic libraries we ship
573 # appear to have this (even when `../lib` is redundant).
575 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
576 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
578 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
579 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
580 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
583 subprocess.check_output([patchelf] + patchelf_args + [fname])
584 except subprocess.CalledProcessError as reason:
585 print("warning: failed to call patchelf:", reason)
588 def rustc_stamp(self):
589 """Return the path for .rustc-stamp
592 >>> rb.build_dir = "build"
593 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
596 return os.path.join(self.bin_root(), '.rustc-stamp')
598 def rustfmt_stamp(self):
599 """Return the path for .rustfmt-stamp
602 >>> rb.build_dir = "build"
603 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
606 return os.path.join(self.bin_root(), '.rustfmt-stamp')
608 def llvm_stamp(self):
609 """Return the path for .rustfmt-stamp
612 >>> rb.build_dir = "build"
613 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
616 return os.path.join(self.llvm_root(), '.llvm-stamp')
619 def program_out_of_date(self, stamp_path, extra=""):
620 """Check if the given program stamp is out of date"""
621 if not os.path.exists(stamp_path) or self.clean:
623 with open(stamp_path, 'r') as stamp:
624 return (self.date + extra) != stamp.read()
627 """Return the binary root directory
630 >>> rb.build_dir = "build"
631 >>> rb.bin_root() == os.path.join("build", "stage0")
634 When the 'build' property is given should be a nested directory:
636 >>> rb.build = "devel"
637 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
640 return os.path.join(self.build_dir, self.build, "stage0")
643 """Return the CI LLVM root directory
646 >>> rb.build_dir = "build"
647 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
650 When the 'build' property is given should be a nested directory:
652 >>> rb.build = "devel"
653 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
656 return os.path.join(self.build_dir, self.build, "ci-llvm")
658 def get_toml(self, key, section=None):
659 """Returns the value of the given key in config.toml, otherwise returns None
662 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
663 >>> rb.get_toml("key2")
666 If the key does not exists, the result is None:
668 >>> rb.get_toml("key3") is None
671 Optionally also matches the section the key appears in
673 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
674 >>> rb.get_toml('key', 'a')
676 >>> rb.get_toml('key', 'b')
678 >>> rb.get_toml('key', 'c') is None
681 >>> rb.config_toml = 'key1 = true'
682 >>> rb.get_toml("key1")
687 for line in self.config_toml.splitlines():
688 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
689 if section_match is not None:
690 cur_section = section_match.group(1)
692 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
693 if match is not None:
694 value = match.group(1)
695 if section is None or section == cur_section:
696 return self.get_string(value) or value.strip()
700 """Return config path for cargo"""
701 return self.program_config('cargo')
704 """Return config path for rustc"""
705 return self.program_config('rustc')
708 """Return config path for rustfmt"""
709 if not self.rustfmt_channel:
711 return self.program_config('rustfmt')
713 def program_config(self, program):
714 """Return config path for the given program
717 >>> rb.config_toml = 'rustc = "rustc"\\n'
718 >>> rb.program_config('rustc')
720 >>> rb.config_toml = ''
721 >>> cargo_path = rb.program_config('cargo')
722 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
726 config = self.get_toml(program)
728 return os.path.expanduser(config)
729 return os.path.join(self.bin_root(), "bin", "{}{}".format(
730 program, self.exe_suffix()))
733 def get_string(line):
734 """Return the value between double quotes
736 >>> RustBuild.get_string(' "devel" ')
738 >>> RustBuild.get_string(" 'devel' ")
740 >>> RustBuild.get_string('devel') is None
742 >>> RustBuild.get_string(' "devel ')
745 start = line.find('"')
747 end = start + 1 + line[start + 1:].find('"')
748 return line[start + 1:end]
749 start = line.find('\'')
751 end = start + 1 + line[start + 1:].find('\'')
752 return line[start + 1:end]
757 """Return a suffix for executables"""
758 if sys.platform == 'win32':
762 def bootstrap_binary(self):
763 """Return the path of the bootstrap binary
766 >>> rb.build_dir = "build"
767 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
768 ... "debug", "bootstrap")
771 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
773 def build_bootstrap(self):
774 """Build bootstrap"""
775 build_dir = os.path.join(self.build_dir, "bootstrap")
776 if self.clean and os.path.exists(build_dir):
777 shutil.rmtree(build_dir)
778 env = os.environ.copy()
779 # `CARGO_BUILD_TARGET` breaks bootstrap build.
780 # See also: <https://github.com/rust-lang/rust/issues/70208>.
781 if "CARGO_BUILD_TARGET" in env:
782 del env["CARGO_BUILD_TARGET"]
783 env["CARGO_TARGET_DIR"] = build_dir
784 env["RUSTC"] = self.rustc()
785 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
786 (os.pathsep + env["LD_LIBRARY_PATH"]) \
787 if "LD_LIBRARY_PATH" in env else ""
788 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
789 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
790 if "DYLD_LIBRARY_PATH" in env else ""
791 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
792 (os.pathsep + env["LIBRARY_PATH"]) \
793 if "LIBRARY_PATH" in env else ""
794 # preserve existing RUSTFLAGS
795 env.setdefault("RUSTFLAGS", "")
796 env["RUSTFLAGS"] += " -Cdebuginfo=2"
798 build_section = "target.{}".format(self.build)
800 if self.get_toml("crt-static", build_section) == "true":
801 target_features += ["+crt-static"]
802 elif self.get_toml("crt-static", build_section) == "false":
803 target_features += ["-crt-static"]
805 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
806 target_linker = self.get_toml("linker", build_section)
807 if target_linker is not None:
808 env["RUSTFLAGS"] += " -C linker=" + target_linker
809 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
810 if self.get_toml("deny-warnings", "rust") != "false":
811 env["RUSTFLAGS"] += " -Dwarnings"
813 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
814 os.pathsep + env["PATH"]
815 if not os.path.isfile(self.cargo()):
816 raise Exception("no cargo executable found at `{}`".format(
818 args = [self.cargo(), "build", "--manifest-path",
819 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
820 for _ in range(1, self.verbose):
821 args.append("--verbose")
822 if self.use_locked_deps:
823 args.append("--locked")
824 if self.use_vendored_sources:
825 args.append("--frozen")
826 run(args, env=env, verbose=self.verbose)
828 def build_triple(self):
829 """Build triple as in LLVM
831 Note that `default_build_triple` is moderately expensive,
832 so use `self.build` where possible.
834 config = self.get_toml('build')
837 return default_build_triple(self.verbose)
839 def check_submodule(self, module, slow_submodules):
840 if not slow_submodules:
841 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
842 cwd=os.path.join(self.rust_root, module),
843 stdout=subprocess.PIPE)
848 def update_submodule(self, module, checked_out, recorded_submodules):
849 module_path = os.path.join(self.rust_root, module)
851 if checked_out is not None:
852 default_encoding = sys.getdefaultencoding()
853 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
854 if recorded_submodules[module] == checked_out:
857 print("Updating submodule", module)
859 run(["git", "submodule", "-q", "sync", module],
860 cwd=self.rust_root, verbose=self.verbose)
862 update_args = ["git", "submodule", "update", "--init", "--recursive"]
863 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
864 update_args.append("--progress")
865 update_args.append(module)
866 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
868 run(["git", "reset", "-q", "--hard"],
869 cwd=module_path, verbose=self.verbose)
870 run(["git", "clean", "-qdfx"],
871 cwd=module_path, verbose=self.verbose)
873 def update_submodules(self):
874 """Update submodules"""
875 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
876 self.get_toml('submodules') == "false":
879 default_encoding = sys.getdefaultencoding()
881 # check the existence and version of 'git' command
882 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
883 self.git_version = distutils.version.LooseVersion(git_version_str)
885 slow_submodules = self.get_toml('fast-submodules') == "false"
888 print('Unconditionally updating all submodules')
890 print('Updating only changed submodules')
891 default_encoding = sys.getdefaultencoding()
892 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
893 ["git", "config", "--file",
894 os.path.join(self.rust_root, ".gitmodules"),
895 "--get-regexp", "path"]
896 ).decode(default_encoding).splitlines()]
897 filtered_submodules = []
898 submodules_names = []
899 llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git"))
900 external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm()
901 llvm_needed = not self.get_toml('codegen-backends', 'rust') \
902 or "llvm" in self.get_toml('codegen-backends', 'rust')
903 for module in submodules:
904 if module.endswith("llvm-project"):
905 # Don't sync the llvm-project submodule if an external LLVM was
906 # provided, if we are downloading LLVM or if the LLVM backend is
907 # not being built. Also, if the submodule has been initialized
908 # already, sync it anyways so that it doesn't mess up contributor
910 if external_llvm_provided or not llvm_needed:
911 if self.get_toml('lld') != 'true' and not llvm_checked_out:
913 check = self.check_submodule(module, slow_submodules)
914 filtered_submodules.append((module, check))
915 submodules_names.append(module)
916 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
917 cwd=self.rust_root, stdout=subprocess.PIPE)
918 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
919 recorded_submodules = {}
920 for data in recorded:
922 recorded_submodules[data[3]] = data[2]
923 for module in filtered_submodules:
924 self.update_submodule(module[0], module[1], recorded_submodules)
925 print("Submodules updated in %.2f seconds" % (time() - start_time))
927 def set_normal_environment(self):
928 """Set download URL for normal environment"""
929 if 'RUSTUP_DIST_SERVER' in os.environ:
930 self._download_url = os.environ['RUSTUP_DIST_SERVER']
932 self._download_url = 'https://static.rust-lang.org'
934 def set_dev_environment(self):
935 """Set download URL for development environment"""
936 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
937 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
939 self._download_url = 'https://dev-static.rust-lang.org'
941 def check_vendored_status(self):
942 """Check that vendoring is configured properly"""
943 vendor_dir = os.path.join(self.rust_root, 'vendor')
944 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
945 if os.environ.get('USER') != os.environ['SUDO_USER']:
946 self.use_vendored_sources = True
947 print('info: looks like you are running this command under `sudo`')
948 print(' and so in order to preserve your $HOME this will now')
949 print(' use vendored sources by default.')
950 if not os.path.exists(vendor_dir):
951 print('error: vendoring required, but vendor directory does not exist.')
952 print(' Run `cargo vendor` without sudo to initialize the '
954 raise Exception("{} not found".format(vendor_dir))
956 if self.use_vendored_sources:
957 if not os.path.exists('.cargo'):
958 os.makedirs('.cargo')
959 with output('.cargo/config') as cargo_config:
961 "[source.crates-io]\n"
962 "replace-with = 'vendored-sources'\n"
963 "registry = 'https://example.com'\n"
965 "[source.vendored-sources]\n"
966 "directory = '{}/vendor'\n"
967 .format(self.rust_root))
969 if os.path.exists('.cargo'):
970 shutil.rmtree('.cargo')
972 def ensure_vendored(self):
973 """Ensure that the vendored sources are available if needed"""
974 vendor_dir = os.path.join(self.rust_root, 'vendor')
975 # Note that this does not handle updating the vendored dependencies if
976 # the rust git repository is updated. Normal development usually does
977 # not use vendoring, so hopefully this isn't too much of a problem.
978 if self.use_vendored_sources and not os.path.exists(vendor_dir):
982 "--sync=./src/tools/rust-analyzer/Cargo.toml",
983 "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml",
984 ], verbose=self.verbose, cwd=self.rust_root)
987 def bootstrap(help_triggered):
988 """Configure, fetch, build and run the initial bootstrap"""
990 # If the user is asking for help, let them know that the whole download-and-build
991 # process has to happen before anything is printed out.
993 print("info: Downloading and building bootstrap before processing --help")
994 print(" command. See src/bootstrap/README.md for help with common")
997 parser = argparse.ArgumentParser(description='Build rust')
998 parser.add_argument('--config')
999 parser.add_argument('--build')
1000 parser.add_argument('--clean', action='store_true')
1001 parser.add_argument('-v', '--verbose', action='count', default=0)
1003 args = [a for a in sys.argv if a != '-h' and a != '--help']
1004 args, _ = parser.parse_known_args(args)
1006 # Configure initial bootstrap
1008 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1009 build.verbose = args.verbose
1010 build.clean = args.clean
1012 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
1014 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
1015 if not toml_path and os.path.exists('config.toml'):
1016 toml_path = 'config.toml'
1019 if not os.path.exists(toml_path):
1020 toml_path = os.path.join(build.rust_root, toml_path)
1022 with open(toml_path) as config:
1023 build.config_toml = config.read()
1025 profile = build.get_toml('profile')
1026 if profile is not None:
1027 include_file = 'config.{}.toml'.format(profile)
1028 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1029 include_path = os.path.join(include_dir, include_file)
1030 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1031 # specific key, so appending our defaults at the end allows the user to override them
1032 with open(include_path) as included_toml:
1033 build.config_toml += os.linesep + included_toml.read()
1035 config_verbose = build.get_toml('verbose', 'build')
1036 if config_verbose is not None:
1037 build.verbose = max(build.verbose, int(config_verbose))
1039 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1041 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1043 build.check_vendored_status()
1045 build_dir = build.get_toml('build-dir', 'build') or 'build'
1046 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1048 data = stage0_data(build.rust_root)
1049 build.date = data['date']
1050 build.rustc_channel = data['rustc']
1052 if "rustfmt" in data:
1053 build.rustfmt_channel = data['rustfmt']
1056 build.set_dev_environment()
1058 build.set_normal_environment()
1060 build.update_submodules()
1062 # Fetch/build the bootstrap
1063 build.build = args.build or build.build_triple()
1064 build.download_stage0()
1066 build.ensure_vendored()
1067 build.build_bootstrap()
1071 args = [build.bootstrap_binary()]
1072 args.extend(sys.argv[1:])
1073 env = os.environ.copy()
1074 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1075 env["BOOTSTRAP_PYTHON"] = sys.executable
1076 env["BUILD_DIR"] = build.build_dir
1077 env["RUSTC_BOOTSTRAP"] = '1'
1079 env["BOOTSTRAP_CONFIG"] = toml_path
1080 run(args, env=env, verbose=build.verbose)
1084 """Entry point for the bootstrap process"""
1087 # x.py help <cmd> ...
1088 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1089 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1092 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1094 bootstrap(help_triggered)
1095 if not help_triggered:
1096 print("Build completed successfully in {}".format(
1097 format_build_time(time() - start_time)))
1098 except (SystemExit, KeyboardInterrupt) as error:
1099 if hasattr(error, 'code') and isinstance(error.code, int):
1100 exit_code = error.code
1104 if not help_triggered:
1105 print("Build completed unsuccessfully in {}".format(
1106 format_build_time(time() - start_time)))
1110 if __name__ == '__main__':