1 from __future__ import absolute_import, division, print_function
5 import distutils.version
19 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
20 temp_path = temp_file.name
21 with tarfile.open(temp_path, "w:xz"):
24 except tarfile.CompressionError:
27 def get(url, path, verbose=False, do_verify=True):
29 sha_url = url + suffix
30 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
31 temp_path = temp_file.name
32 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
33 sha_path = sha_file.name
37 download(sha_path, sha_url, False, verbose)
38 if os.path.exists(path):
39 if verify(path, sha_path, False):
41 print("using already-download file", path)
45 print("ignoring already-download file",
46 path, "due to failed verification")
48 download(temp_path, url, True, verbose)
49 if do_verify and not verify(temp_path, sha_path, verbose):
50 raise RuntimeError("failed verification")
52 print("moving {} to {}".format(temp_path, path))
53 shutil.move(temp_path, path)
55 delete_if_present(sha_path, verbose)
56 delete_if_present(temp_path, verbose)
59 def delete_if_present(path, verbose):
60 """Remove the given file if present"""
61 if os.path.isfile(path):
63 print("removing", path)
67 def download(path, url, probably_big, verbose):
70 _download(path, url, probably_big, verbose, True)
73 print("\nspurious failure, trying again")
74 _download(path, url, probably_big, verbose, False)
77 def _download(path, url, probably_big, verbose, exception):
78 if probably_big or verbose:
79 print("downloading {}".format(url))
80 # see http://serverfault.com/questions/301128/how-to-download
81 if sys.platform == 'win32':
82 run(["PowerShell.exe", "/nologo", "-Command",
83 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
84 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
88 if probably_big or verbose:
92 require(["curl", "--version"])
94 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
95 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
96 "--retry", "3", "-Sf", "-o", path, url],
101 def verify(path, sha_path, verbose):
102 """Check if the sha256 sum of the given path is valid"""
104 print("verifying", path)
105 with open(path, "rb") as source:
106 found = hashlib.sha256(source.read()).hexdigest()
107 with open(sha_path, "r") as sha256sum:
108 expected = sha256sum.readline().split()[0]
109 verified = found == expected
111 print("invalid checksum:\n"
113 " expected: {}".format(found, expected))
117 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
118 """Unpack the given tarball file"""
119 print("extracting", tarball)
120 fname = os.path.basename(tarball).replace(tarball_suffix, "")
121 with contextlib.closing(tarfile.open(tarball)) as tar:
122 for member in tar.getnames():
123 if "/" not in member:
125 name = member.replace(fname + "/", "", 1)
126 if match is not None and not name.startswith(match):
128 name = name[len(match) + 1:]
130 dst_path = os.path.join(dst, name)
132 print(" extracting", member)
133 tar.extract(member, dst)
134 src_path = os.path.join(dst, member)
135 if os.path.isdir(src_path) and os.path.exists(dst_path):
137 shutil.move(src_path, dst_path)
138 shutil.rmtree(os.path.join(dst, fname))
141 def run(args, verbose=False, exception=False, **kwargs):
142 """Run a child program in a new process"""
144 print("running: " + ' '.join(args))
146 # Use Popen here instead of call() as it apparently allows powershell on
147 # Windows to not lock up waiting for input presumably.
148 ret = subprocess.Popen(args, **kwargs)
151 err = "failed to run: " + ' '.join(args)
152 if verbose or exception:
153 raise RuntimeError(err)
157 def require(cmd, exit=True):
158 '''Run a command, returning its output.
160 If `exit` is `True`, exit the process.
161 Otherwise, return None.'''
163 return subprocess.check_output(cmd).strip()
164 except (subprocess.CalledProcessError, OSError) as exc:
167 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
168 print("Please make sure it's installed and in the path.")
172 def stage0_data(rust_root):
173 """Build a dictionary from stage0.txt"""
174 nightlies = os.path.join(rust_root, "src/stage0.txt")
175 with open(nightlies, 'r') as nightlies:
176 lines = [line.rstrip() for line in nightlies
177 if not line.startswith("#")]
178 return dict([line.split(": ", 1) for line in lines if line])
181 def format_build_time(duration):
182 """Return a nicer format for build time
184 >>> format_build_time('300')
187 return str(datetime.timedelta(seconds=int(duration)))
190 def default_build_triple():
191 """Build triple as in LLVM"""
192 default_encoding = sys.getdefaultencoding()
193 required = sys.platform != 'win32'
194 ostype = require(["uname", "-s"], exit=required)
195 cputype = require(['uname', '-m'], exit=required)
197 # If we do not have `uname`, assume Windows.
198 if ostype is None or cputype is None:
199 return 'x86_64-pc-windows-msvc'
201 ostype = ostype.decode(default_encoding)
202 cputype = cputype.decode(default_encoding)
204 # The goal here is to come up with the same triple as LLVM would,
205 # at least for the subset of platforms we're willing to target.
207 'Darwin': 'apple-darwin',
208 'DragonFly': 'unknown-dragonfly',
209 'FreeBSD': 'unknown-freebsd',
210 'Haiku': 'unknown-haiku',
211 'NetBSD': 'unknown-netbsd',
212 'OpenBSD': 'unknown-openbsd'
215 # Consider the direct transformation first and then the special cases
216 if ostype in ostype_mapper:
217 ostype = ostype_mapper[ostype]
218 elif ostype == 'Linux':
219 os_from_sp = subprocess.check_output(
220 ['uname', '-o']).strip().decode(default_encoding)
221 if os_from_sp == 'Android':
222 ostype = 'linux-android'
224 ostype = 'unknown-linux-gnu'
225 elif ostype == 'SunOS':
226 ostype = 'sun-solaris'
227 # On Solaris, uname -m will return a machine classification instead
228 # of a cpu type, so uname -p is recommended instead. However, the
229 # output from that option is too generic for our purposes (it will
230 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
231 # must be used instead.
232 cputype = require(['isainfo', '-k']).decode(default_encoding)
233 elif ostype.startswith('MINGW'):
234 # msys' `uname` does not print gcc configuration, but prints msys
235 # configuration. so we cannot believe `uname -m`:
236 # msys1 is always i686 and msys2 is always x86_64.
237 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
239 ostype = 'pc-windows-gnu'
241 if os.environ.get('MSYSTEM') == 'MINGW64':
243 elif ostype.startswith('MSYS'):
244 ostype = 'pc-windows-gnu'
245 elif ostype.startswith('CYGWIN_NT'):
247 if ostype.endswith('WOW64'):
249 ostype = 'pc-windows-gnu'
250 elif sys.platform == 'win32':
251 # Some Windows platforms might have a `uname` command that returns a
252 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
253 # these cases, fall back to using sys.platform.
254 return 'x86_64-pc-windows-msvc'
256 err = "unknown OS type: {}".format(ostype)
259 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
260 cputype = subprocess.check_output(
261 ['uname', '-p']).strip().decode(default_encoding)
264 'aarch64': 'aarch64',
271 'powerpc': 'powerpc',
272 'powerpc64': 'powerpc64',
273 'powerpc64le': 'powerpc64le',
275 'ppc64': 'powerpc64',
276 'ppc64le': 'powerpc64le',
284 # Consider the direct transformation first and then the special cases
285 if cputype in cputype_mapper:
286 cputype = cputype_mapper[cputype]
287 elif cputype in {'xscale', 'arm'}:
289 if ostype == 'linux-android':
290 ostype = 'linux-androideabi'
291 elif ostype == 'unknown-freebsd':
292 cputype = subprocess.check_output(
293 ['uname', '-p']).strip().decode(default_encoding)
294 ostype = 'unknown-freebsd'
295 elif cputype == 'armv6l':
297 if ostype == 'linux-android':
298 ostype = 'linux-androideabi'
301 elif cputype in {'armv7l', 'armv8l'}:
303 if ostype == 'linux-android':
304 ostype = 'linux-androideabi'
307 elif cputype == 'mips':
308 if sys.byteorder == 'big':
310 elif sys.byteorder == 'little':
313 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
314 elif cputype == 'mips64':
315 if sys.byteorder == 'big':
317 elif sys.byteorder == 'little':
320 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
321 # only the n64 ABI is supported, indicate it
323 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
326 err = "unknown cpu type: {}".format(cputype)
329 return "{}-{}".format(cputype, ostype)
332 @contextlib.contextmanager
333 def output(filepath):
334 tmp = filepath + '.tmp'
335 with open(tmp, 'w') as f:
338 os.remove(filepath) # PermissionError/OSError on Win32 if in use
339 os.rename(tmp, filepath)
341 shutil.copy2(tmp, filepath)
345 class RustBuild(object):
346 """Provide all the methods required to build Rust"""
348 self.cargo_channel = ''
350 self._download_url = ''
351 self.rustc_channel = ''
352 self.rustfmt_channel = ''
356 self.config_toml = ''
358 self.use_locked_deps = ''
359 self.use_vendored_sources = ''
361 self.git_version = None
362 self.nix_deps_dir = None
364 def download_stage0(self):
365 """Fetch the build system for Rust, written in Rust
367 This method will build a cache directory, then it will fetch the
368 tarball which has the stage0 compiler used to then bootstrap the Rust
371 Each downloaded tarball is extracted, after that, the script
372 will move all the content to the right place.
374 rustc_channel = self.rustc_channel
375 cargo_channel = self.cargo_channel
376 rustfmt_channel = self.rustfmt_channel
378 if self.rustc().startswith(self.bin_root()) and \
379 (not os.path.exists(self.rustc()) or
380 self.program_out_of_date(self.rustc_stamp())):
381 if os.path.exists(self.bin_root()):
382 shutil.rmtree(self.bin_root())
383 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
384 filename = "rust-std-{}-{}{}".format(
385 rustc_channel, self.build, tarball_suffix)
386 pattern = "rust-std-{}".format(self.build)
387 self._download_stage0_helper(filename, pattern, tarball_suffix)
389 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
391 self._download_stage0_helper(filename, "rustc", tarball_suffix)
392 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
393 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
394 lib_dir = "{}/lib".format(self.bin_root())
395 for lib in os.listdir(lib_dir):
396 if lib.endswith(".so"):
397 self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
398 with output(self.rustc_stamp()) as rust_stamp:
399 rust_stamp.write(self.date)
401 if self.cargo().startswith(self.bin_root()) and \
402 (not os.path.exists(self.cargo()) or
403 self.program_out_of_date(self.cargo_stamp())):
404 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
405 filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
407 self._download_stage0_helper(filename, "cargo", tarball_suffix)
408 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
409 with output(self.cargo_stamp()) as cargo_stamp:
410 cargo_stamp.write(self.date)
412 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
413 not os.path.exists(self.rustfmt())
414 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
417 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
418 [channel, date] = rustfmt_channel.split('-', 1)
419 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
420 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
421 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
422 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
423 with output(self.rustfmt_stamp()) as rustfmt_stamp:
424 rustfmt_stamp.write(self.date + self.rustfmt_channel)
426 if self.downloading_llvm():
427 # We want the most recent LLVM submodule update to avoid downloading
428 # LLVM more often than necessary.
430 # This git command finds that commit SHA, looking for bors-authored
431 # merges that modified src/llvm-project.
433 # This works even in a repository that has not yet initialized
435 llvm_sha = subprocess.check_output([
436 "git", "log", "--author=bors", "--format=%H", "-n1",
437 "-m", "--first-parent",
440 "src/bootstrap/download-ci-llvm-stamp",
441 ]).decode(sys.getdefaultencoding()).strip()
442 llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
443 if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
444 self._download_ci_llvm(llvm_sha, llvm_assertions)
445 for binary in ["llvm-config", "FileCheck"]:
446 self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary))
447 with output(self.llvm_stamp()) as llvm_stamp:
448 llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions))
450 def downloading_llvm(self):
451 opt = self.get_toml('download-ci-llvm', 'llvm')
452 return opt == "true" \
453 or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
455 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
458 cache_dst = os.path.join(self.build_dir, "cache")
459 rustc_cache = os.path.join(cache_dst, date)
460 if not os.path.exists(rustc_cache):
461 os.makedirs(rustc_cache)
463 url = "{}/dist/{}".format(self._download_url, date)
464 tarball = os.path.join(rustc_cache, filename)
465 if not os.path.exists(tarball):
466 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
467 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
469 def _download_ci_llvm(self, llvm_sha, llvm_assertions):
470 cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
471 cache_dst = os.path.join(self.build_dir, "cache")
472 rustc_cache = os.path.join(cache_dst, cache_prefix)
473 if not os.path.exists(rustc_cache):
474 os.makedirs(rustc_cache)
476 url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha)
478 url = url.replace('rustc-builds', 'rustc-builds-alt')
479 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
480 filename = "rust-dev-nightly-" + self.build + tarball_suffix
481 tarball = os.path.join(rustc_cache, filename)
482 if not os.path.exists(tarball):
483 get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False)
484 unpack(tarball, tarball_suffix, self.llvm_root(),
486 verbose=self.verbose)
488 def fix_bin_or_dylib(self, fname):
489 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
490 or the RPATH section, to fix the dynamic library search path
492 This method is only required on NixOS and uses the PatchELF utility to
493 change the interpreter/RPATH of ELF executables.
495 Please see https://nixos.org/patchelf.html for more information
497 default_encoding = sys.getdefaultencoding()
499 ostype = subprocess.check_output(
500 ['uname', '-s']).strip().decode(default_encoding)
501 except subprocess.CalledProcessError:
503 except OSError as reason:
504 if getattr(reason, 'winerror', None) is not None:
508 if ostype != "Linux":
511 if not os.path.exists("/etc/NIXOS"):
513 if os.path.exists("/lib"):
516 # At this point we're pretty sure the user is running NixOS
517 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
518 print(nix_os_msg, fname)
520 # Only build `stage0/.nix-deps` once.
521 nix_deps_dir = self.nix_deps_dir
523 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
524 if not os.path.exists(nix_deps_dir):
525 os.makedirs(nix_deps_dir)
528 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
529 "stdenv.cc.bintools",
531 # Needed as a system dependency of `libLLVM-*.so`.
534 # Needed for patching ELF binaries (see doc comment above).
538 # Run `nix-build` to "build" each dependency (which will likely reuse
539 # the existing `/nix/store` copy, or at most download a pre-built copy).
540 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
541 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
542 # ensuring garbage collection will never remove the `/nix/store` path
543 # (which would break our patched binaries that hardcode those paths).
546 subprocess.check_output([
547 "nix-build", "<nixpkgs>",
549 "-o", "{}/{}".format(nix_deps_dir, dep),
551 except subprocess.CalledProcessError as reason:
552 print("warning: failed to call nix-build:", reason)
555 self.nix_deps_dir = nix_deps_dir
557 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
559 if fname.endswith(".so"):
560 # Dynamic library, patch RPATH to point to system dependencies.
561 dylib_deps = ["zlib"]
563 # Relative default, all binary and dynamic libraries we ship
564 # appear to have this (even when `../lib` is redundant).
566 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
567 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
569 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
570 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
571 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
574 subprocess.check_output([patchelf] + patchelf_args + [fname])
575 except subprocess.CalledProcessError as reason:
576 print("warning: failed to call patchelf:", reason)
579 def rustc_stamp(self):
580 """Return the path for .rustc-stamp
583 >>> rb.build_dir = "build"
584 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
587 return os.path.join(self.bin_root(), '.rustc-stamp')
589 def cargo_stamp(self):
590 """Return the path for .cargo-stamp
593 >>> rb.build_dir = "build"
594 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
597 return os.path.join(self.bin_root(), '.cargo-stamp')
599 def rustfmt_stamp(self):
600 """Return the path for .rustfmt-stamp
603 >>> rb.build_dir = "build"
604 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
607 return os.path.join(self.bin_root(), '.rustfmt-stamp')
609 def llvm_stamp(self):
610 """Return the path for .rustfmt-stamp
613 >>> rb.build_dir = "build"
614 >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
617 return os.path.join(self.llvm_root(), '.llvm-stamp')
620 def program_out_of_date(self, stamp_path, extra=""):
621 """Check if the given program stamp is out of date"""
622 if not os.path.exists(stamp_path) or self.clean:
624 with open(stamp_path, 'r') as stamp:
625 return (self.date + extra) != stamp.read()
628 """Return the binary root directory
631 >>> rb.build_dir = "build"
632 >>> rb.bin_root() == os.path.join("build", "stage0")
635 When the 'build' property is given should be a nested directory:
637 >>> rb.build = "devel"
638 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
641 return os.path.join(self.build_dir, self.build, "stage0")
644 """Return the CI LLVM root directory
647 >>> rb.build_dir = "build"
648 >>> rb.llvm_root() == os.path.join("build", "ci-llvm")
651 When the 'build' property is given should be a nested directory:
653 >>> rb.build = "devel"
654 >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
657 return os.path.join(self.build_dir, self.build, "ci-llvm")
659 def get_toml(self, key, section=None):
660 """Returns the value of the given key in config.toml, otherwise returns None
663 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
664 >>> rb.get_toml("key2")
667 If the key does not exists, the result is None:
669 >>> rb.get_toml("key3") is None
672 Optionally also matches the section the key appears in
674 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
675 >>> rb.get_toml('key', 'a')
677 >>> rb.get_toml('key', 'b')
679 >>> rb.get_toml('key', 'c') is None
682 >>> rb.config_toml = 'key1 = true'
683 >>> rb.get_toml("key1")
688 for line in self.config_toml.splitlines():
689 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
690 if section_match is not None:
691 cur_section = section_match.group(1)
693 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
694 if match is not None:
695 value = match.group(1)
696 if section is None or section == cur_section:
697 return self.get_string(value) or value.strip()
701 """Return config path for cargo"""
702 return self.program_config('cargo')
705 """Return config path for rustc"""
706 return self.program_config('rustc')
709 """Return config path for rustfmt"""
710 if not self.rustfmt_channel:
712 return self.program_config('rustfmt')
714 def program_config(self, program):
715 """Return config path for the given program
718 >>> rb.config_toml = 'rustc = "rustc"\\n'
719 >>> rb.program_config('rustc')
721 >>> rb.config_toml = ''
722 >>> cargo_path = rb.program_config('cargo')
723 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
727 config = self.get_toml(program)
729 return os.path.expanduser(config)
730 return os.path.join(self.bin_root(), "bin", "{}{}".format(
731 program, self.exe_suffix()))
734 def get_string(line):
735 """Return the value between double quotes
737 >>> RustBuild.get_string(' "devel" ')
739 >>> RustBuild.get_string(" 'devel' ")
741 >>> RustBuild.get_string('devel') is None
743 >>> RustBuild.get_string(' "devel ')
746 start = line.find('"')
748 end = start + 1 + line[start + 1:].find('"')
749 return line[start + 1:end]
750 start = line.find('\'')
752 end = start + 1 + line[start + 1:].find('\'')
753 return line[start + 1:end]
758 """Return a suffix for executables"""
759 if sys.platform == 'win32':
763 def bootstrap_binary(self):
764 """Return the path of the bootstrap binary
767 >>> rb.build_dir = "build"
768 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
769 ... "debug", "bootstrap")
772 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
774 def build_bootstrap(self):
775 """Build bootstrap"""
776 build_dir = os.path.join(self.build_dir, "bootstrap")
777 if self.clean and os.path.exists(build_dir):
778 shutil.rmtree(build_dir)
779 env = os.environ.copy()
780 # `CARGO_BUILD_TARGET` breaks bootstrap build.
781 # See also: <https://github.com/rust-lang/rust/issues/70208>.
782 if "CARGO_BUILD_TARGET" in env:
783 del env["CARGO_BUILD_TARGET"]
784 env["CARGO_TARGET_DIR"] = build_dir
785 env["RUSTC"] = self.rustc()
786 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
787 (os.pathsep + env["LD_LIBRARY_PATH"]) \
788 if "LD_LIBRARY_PATH" in env else ""
789 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
790 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
791 if "DYLD_LIBRARY_PATH" in env else ""
792 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
793 (os.pathsep + env["LIBRARY_PATH"]) \
794 if "LIBRARY_PATH" in env else ""
795 # preserve existing RUSTFLAGS
796 env.setdefault("RUSTFLAGS", "")
797 env["RUSTFLAGS"] += " -Cdebuginfo=2"
799 build_section = "target.{}".format(self.build_triple())
801 if self.get_toml("crt-static", build_section) == "true":
802 target_features += ["+crt-static"]
803 elif self.get_toml("crt-static", build_section) == "false":
804 target_features += ["-crt-static"]
806 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
807 target_linker = self.get_toml("linker", build_section)
808 if target_linker is not None:
809 env["RUSTFLAGS"] += " -C linker=" + target_linker
810 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
811 if self.get_toml("deny-warnings", "rust") != "false":
812 env["RUSTFLAGS"] += " -Dwarnings"
814 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
815 os.pathsep + env["PATH"]
816 if not os.path.isfile(self.cargo()):
817 raise Exception("no cargo executable found at `{}`".format(
819 args = [self.cargo(), "build", "--manifest-path",
820 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
821 for _ in range(1, self.verbose):
822 args.append("--verbose")
823 if self.use_locked_deps:
824 args.append("--locked")
825 if self.use_vendored_sources:
826 args.append("--frozen")
827 run(args, env=env, verbose=self.verbose)
829 def build_triple(self):
830 """Build triple as in LLVM"""
831 config = self.get_toml('build')
834 return default_build_triple()
836 def check_submodule(self, module, slow_submodules):
837 if not slow_submodules:
838 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
839 cwd=os.path.join(self.rust_root, module),
840 stdout=subprocess.PIPE)
845 def update_submodule(self, module, checked_out, recorded_submodules):
846 module_path = os.path.join(self.rust_root, module)
848 if checked_out is not None:
849 default_encoding = sys.getdefaultencoding()
850 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
851 if recorded_submodules[module] == checked_out:
854 print("Updating submodule", module)
856 run(["git", "submodule", "-q", "sync", module],
857 cwd=self.rust_root, verbose=self.verbose)
859 update_args = ["git", "submodule", "update", "--init", "--recursive"]
860 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
861 update_args.append("--progress")
862 update_args.append(module)
863 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
865 run(["git", "reset", "-q", "--hard"],
866 cwd=module_path, verbose=self.verbose)
867 run(["git", "clean", "-qdfx"],
868 cwd=module_path, verbose=self.verbose)
870 def update_submodules(self):
871 """Update submodules"""
872 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
873 self.get_toml('submodules') == "false":
876 default_encoding = sys.getdefaultencoding()
878 # check the existence and version of 'git' command
879 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
880 self.git_version = distutils.version.LooseVersion(git_version_str)
882 slow_submodules = self.get_toml('fast-submodules') == "false"
885 print('Unconditionally updating all submodules')
887 print('Updating only changed submodules')
888 default_encoding = sys.getdefaultencoding()
889 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
890 ["git", "config", "--file",
891 os.path.join(self.rust_root, ".gitmodules"),
892 "--get-regexp", "path"]
893 ).decode(default_encoding).splitlines()]
894 filtered_submodules = []
895 submodules_names = []
896 for module in submodules:
897 if module.endswith("llvm-project"):
898 if self.get_toml('llvm-config') or self.downloading_llvm():
899 if self.get_toml('lld') != 'true':
901 check = self.check_submodule(module, slow_submodules)
902 filtered_submodules.append((module, check))
903 submodules_names.append(module)
904 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
905 cwd=self.rust_root, stdout=subprocess.PIPE)
906 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
907 recorded_submodules = {}
908 for data in recorded:
910 recorded_submodules[data[3]] = data[2]
911 for module in filtered_submodules:
912 self.update_submodule(module[0], module[1], recorded_submodules)
913 print("Submodules updated in %.2f seconds" % (time() - start_time))
915 def set_normal_environment(self):
916 """Set download URL for normal environment"""
917 if 'RUSTUP_DIST_SERVER' in os.environ:
918 self._download_url = os.environ['RUSTUP_DIST_SERVER']
920 self._download_url = 'https://static.rust-lang.org'
922 def set_dev_environment(self):
923 """Set download URL for development environment"""
924 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
925 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
927 self._download_url = 'https://dev-static.rust-lang.org'
929 def check_vendored_status(self):
930 """Check that vendoring is configured properly"""
931 vendor_dir = os.path.join(self.rust_root, 'vendor')
932 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
933 if os.environ.get('USER') != os.environ['SUDO_USER']:
934 self.use_vendored_sources = True
935 print('info: looks like you are running this command under `sudo`')
936 print(' and so in order to preserve your $HOME this will now')
937 print(' use vendored sources by default.')
938 if not os.path.exists(vendor_dir):
939 print('error: vendoring required, but vendor directory does not exist.')
940 print(' Run `cargo vendor` without sudo to initialize the '
942 raise Exception("{} not found".format(vendor_dir))
944 if self.use_vendored_sources:
945 if not os.path.exists('.cargo'):
946 os.makedirs('.cargo')
947 with output('.cargo/config') as cargo_config:
949 "[source.crates-io]\n"
950 "replace-with = 'vendored-sources'\n"
951 "registry = 'https://example.com'\n"
953 "[source.vendored-sources]\n"
954 "directory = '{}/vendor'\n"
955 .format(self.rust_root))
957 if os.path.exists('.cargo'):
958 shutil.rmtree('.cargo')
960 def ensure_vendored(self):
961 """Ensure that the vendored sources are available if needed"""
962 vendor_dir = os.path.join(self.rust_root, 'vendor')
963 # Note that this does not handle updating the vendored dependencies if
964 # the rust git repository is updated. Normal development usually does
965 # not use vendoring, so hopefully this isn't too much of a problem.
966 if self.use_vendored_sources and not os.path.exists(vendor_dir):
967 run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
968 verbose=self.verbose, cwd=self.rust_root)
971 def bootstrap(help_triggered):
972 """Configure, fetch, build and run the initial bootstrap"""
974 # If the user is asking for help, let them know that the whole download-and-build
975 # process has to happen before anything is printed out.
977 print("info: Downloading and building bootstrap before processing --help")
978 print(" command. See src/bootstrap/README.md for help with common")
981 parser = argparse.ArgumentParser(description='Build rust')
982 parser.add_argument('--config')
983 parser.add_argument('--build')
984 parser.add_argument('--clean', action='store_true')
985 parser.add_argument('-v', '--verbose', action='count', default=0)
987 args = [a for a in sys.argv if a != '-h' and a != '--help']
988 args, _ = parser.parse_known_args(args)
990 # Configure initial bootstrap
992 build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
993 build.verbose = args.verbose
994 build.clean = args.clean
996 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
998 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
999 if not toml_path and os.path.exists('config.toml'):
1000 toml_path = 'config.toml'
1003 if not os.path.exists(toml_path):
1004 toml_path = os.path.join(build.rust_root, toml_path)
1006 with open(toml_path) as config:
1007 build.config_toml = config.read()
1009 profile = build.get_toml('profile')
1010 if profile is not None:
1011 include_file = 'config.{}.toml'.format(profile)
1012 include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
1013 include_path = os.path.join(include_dir, include_file)
1014 # HACK: This works because `build.get_toml()` returns the first match it finds for a
1015 # specific key, so appending our defaults at the end allows the user to override them
1016 with open(include_path) as included_toml:
1017 build.config_toml += os.linesep + included_toml.read()
1019 config_verbose = build.get_toml('verbose', 'build')
1020 if config_verbose is not None:
1021 build.verbose = max(build.verbose, int(config_verbose))
1023 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
1025 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
1027 build.check_vendored_status()
1029 build_dir = build.get_toml('build-dir', 'build') or 'build'
1030 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
1032 data = stage0_data(build.rust_root)
1033 build.date = data['date']
1034 build.rustc_channel = data['rustc']
1035 build.cargo_channel = data['cargo']
1037 if "rustfmt" in data:
1038 build.rustfmt_channel = data['rustfmt']
1041 build.set_dev_environment()
1043 build.set_normal_environment()
1045 build.update_submodules()
1047 # Fetch/build the bootstrap
1048 build.build = args.build or build.build_triple()
1049 build.download_stage0()
1051 build.ensure_vendored()
1052 build.build_bootstrap()
1056 args = [build.bootstrap_binary()]
1057 args.extend(sys.argv[1:])
1058 env = os.environ.copy()
1059 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1060 env["BOOTSTRAP_PYTHON"] = sys.executable
1061 env["BUILD_DIR"] = build.build_dir
1062 env["RUSTC_BOOTSTRAP"] = '1'
1064 env["BOOTSTRAP_CONFIG"] = toml_path
1065 run(args, env=env, verbose=build.verbose)
1069 """Entry point for the bootstrap process"""
1072 # x.py help <cmd> ...
1073 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1074 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1077 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1079 bootstrap(help_triggered)
1080 if not help_triggered:
1081 print("Build completed successfully in {}".format(
1082 format_build_time(time() - start_time)))
1083 except (SystemExit, KeyboardInterrupt) as error:
1084 if hasattr(error, 'code') and isinstance(error.code, int):
1085 exit_code = error.code
1089 if not help_triggered:
1090 print("Build completed unsuccessfully in {}".format(
1091 format_build_time(time() - start_time)))
1095 if __name__ == '__main__':