1 from __future__ import absolute_import, division, print_function
5 import distutils.version
18 def get(url, path, verbose=False):
20 sha_url = url + suffix
21 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
22 temp_path = temp_file.name
23 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
24 sha_path = sha_file.name
27 download(sha_path, sha_url, False, verbose)
28 if os.path.exists(path):
29 if verify(path, sha_path, False):
31 print("using already-download file", path)
35 print("ignoring already-download file",
36 path, "due to failed verification")
38 download(temp_path, url, True, verbose)
39 if not verify(temp_path, sha_path, verbose):
40 raise RuntimeError("failed verification")
42 print("moving {} to {}".format(temp_path, path))
43 shutil.move(temp_path, path)
45 delete_if_present(sha_path, verbose)
46 delete_if_present(temp_path, verbose)
49 def delete_if_present(path, verbose):
50 """Remove the given file if present"""
51 if os.path.isfile(path):
53 print("removing", path)
57 def download(path, url, probably_big, verbose):
60 _download(path, url, probably_big, verbose, True)
63 print("\nspurious failure, trying again")
64 _download(path, url, probably_big, verbose, False)
67 def _download(path, url, probably_big, verbose, exception):
68 if probably_big or verbose:
69 print("downloading {}".format(url))
70 # see http://serverfault.com/questions/301128/how-to-download
71 if sys.platform == 'win32':
72 run(["PowerShell.exe", "/nologo", "-Command",
73 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
74 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
78 if probably_big or verbose:
82 require(["curl", "--version"])
84 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds
85 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
86 "--retry", "3", "-Sf", "-o", path, url],
91 def verify(path, sha_path, verbose):
92 """Check if the sha256 sum of the given path is valid"""
94 print("verifying", path)
95 with open(path, "rb") as source:
96 found = hashlib.sha256(source.read()).hexdigest()
97 with open(sha_path, "r") as sha256sum:
98 expected = sha256sum.readline().split()[0]
99 verified = found == expected
101 print("invalid checksum:\n"
103 " expected: {}".format(found, expected))
107 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
108 """Unpack the given tarball file"""
109 print("extracting", tarball)
110 fname = os.path.basename(tarball).replace(tarball_suffix, "")
111 with contextlib.closing(tarfile.open(tarball)) as tar:
112 for member in tar.getnames():
113 if "/" not in member:
115 name = member.replace(fname + "/", "", 1)
116 if match is not None and not name.startswith(match):
118 name = name[len(match) + 1:]
120 dst_path = os.path.join(dst, name)
122 print(" extracting", member)
123 tar.extract(member, dst)
124 src_path = os.path.join(dst, member)
125 if os.path.isdir(src_path) and os.path.exists(dst_path):
127 shutil.move(src_path, dst_path)
128 shutil.rmtree(os.path.join(dst, fname))
131 def run(args, verbose=False, exception=False, **kwargs):
132 """Run a child program in a new process"""
134 print("running: " + ' '.join(args))
136 # Use Popen here instead of call() as it apparently allows powershell on
137 # Windows to not lock up waiting for input presumably.
138 ret = subprocess.Popen(args, **kwargs)
141 err = "failed to run: " + ' '.join(args)
142 if verbose or exception:
143 raise RuntimeError(err)
147 def require(cmd, exit=True):
148 '''Run a command, returning its output.
150 If `exit` is `True`, exit the process.
151 Otherwise, return None.'''
153 return subprocess.check_output(cmd).strip()
154 except (subprocess.CalledProcessError, OSError) as exc:
157 print("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
158 print("Please make sure it's installed and in the path.")
162 def stage0_data(rust_root):
163 """Build a dictionary from stage0.txt"""
164 nightlies = os.path.join(rust_root, "src/stage0.txt")
165 with open(nightlies, 'r') as nightlies:
166 lines = [line.rstrip() for line in nightlies
167 if not line.startswith("#")]
168 return dict([line.split(": ", 1) for line in lines if line])
171 def format_build_time(duration):
172 """Return a nicer format for build time
174 >>> format_build_time('300')
177 return str(datetime.timedelta(seconds=int(duration)))
180 def default_build_triple():
181 """Build triple as in LLVM"""
182 default_encoding = sys.getdefaultencoding()
183 required = sys.platform != 'win32'
184 ostype = require(["uname", "-s"], exit=required)
185 cputype = require(['uname', '-m'], exit=required)
187 # If we do not have `uname`, assume Windows.
188 if ostype is None or cputype is None:
189 return 'x86_64-pc-windows-msvc'
191 ostype = ostype.decode(default_encoding)
192 cputype = cputype.decode(default_encoding)
194 # The goal here is to come up with the same triple as LLVM would,
195 # at least for the subset of platforms we're willing to target.
197 'Darwin': 'apple-darwin',
198 'DragonFly': 'unknown-dragonfly',
199 'FreeBSD': 'unknown-freebsd',
200 'Haiku': 'unknown-haiku',
201 'NetBSD': 'unknown-netbsd',
202 'OpenBSD': 'unknown-openbsd'
205 # Consider the direct transformation first and then the special cases
206 if ostype in ostype_mapper:
207 ostype = ostype_mapper[ostype]
208 elif ostype == 'Linux':
209 os_from_sp = subprocess.check_output(
210 ['uname', '-o']).strip().decode(default_encoding)
211 if os_from_sp == 'Android':
212 ostype = 'linux-android'
214 ostype = 'unknown-linux-gnu'
215 elif ostype == 'SunOS':
216 ostype = 'sun-solaris'
217 # On Solaris, uname -m will return a machine classification instead
218 # of a cpu type, so uname -p is recommended instead. However, the
219 # output from that option is too generic for our purposes (it will
220 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
221 # must be used instead.
222 cputype = require(['isainfo', '-k']).decode(default_encoding)
223 elif ostype.startswith('MINGW'):
224 # msys' `uname` does not print gcc configuration, but prints msys
225 # configuration. so we cannot believe `uname -m`:
226 # msys1 is always i686 and msys2 is always x86_64.
227 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
229 ostype = 'pc-windows-gnu'
231 if os.environ.get('MSYSTEM') == 'MINGW64':
233 elif ostype.startswith('MSYS'):
234 ostype = 'pc-windows-gnu'
235 elif ostype.startswith('CYGWIN_NT'):
237 if ostype.endswith('WOW64'):
239 ostype = 'pc-windows-gnu'
240 elif sys.platform == 'win32':
241 # Some Windows platforms might have a `uname` command that returns a
242 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
243 # these cases, fall back to using sys.platform.
244 return 'x86_64-pc-windows-msvc'
246 err = "unknown OS type: {}".format(ostype)
249 if cputype == 'powerpc' and ostype == 'unknown-freebsd':
250 cputype = subprocess.check_output(
251 ['uname', '-p']).strip().decode(default_encoding)
254 'aarch64': 'aarch64',
261 'powerpc': 'powerpc',
262 'powerpc64': 'powerpc64',
263 'powerpc64le': 'powerpc64le',
265 'ppc64': 'powerpc64',
266 'ppc64le': 'powerpc64le',
274 # Consider the direct transformation first and then the special cases
275 if cputype in cputype_mapper:
276 cputype = cputype_mapper[cputype]
277 elif cputype in {'xscale', 'arm'}:
279 if ostype == 'linux-android':
280 ostype = 'linux-androideabi'
281 elif ostype == 'unknown-freebsd':
282 cputype = subprocess.check_output(
283 ['uname', '-p']).strip().decode(default_encoding)
284 ostype = 'unknown-freebsd'
285 elif cputype == 'armv6l':
287 if ostype == 'linux-android':
288 ostype = 'linux-androideabi'
291 elif cputype in {'armv7l', 'armv8l'}:
293 if ostype == 'linux-android':
294 ostype = 'linux-androideabi'
297 elif cputype == 'mips':
298 if sys.byteorder == 'big':
300 elif sys.byteorder == 'little':
303 raise ValueError("unknown byteorder: {}".format(sys.byteorder))
304 elif cputype == 'mips64':
305 if sys.byteorder == 'big':
307 elif sys.byteorder == 'little':
310 raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
311 # only the n64 ABI is supported, indicate it
313 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
316 err = "unknown cpu type: {}".format(cputype)
319 return "{}-{}".format(cputype, ostype)
322 @contextlib.contextmanager
323 def output(filepath):
324 tmp = filepath + '.tmp'
325 with open(tmp, 'w') as f:
328 os.remove(filepath) # PermissionError/OSError on Win32 if in use
329 os.rename(tmp, filepath)
331 shutil.copy2(tmp, filepath)
335 class RustBuild(object):
336 """Provide all the methods required to build Rust"""
338 self.cargo_channel = ''
340 self._download_url = ''
341 self.rustc_channel = ''
342 self.rustfmt_channel = ''
346 self.config_toml = ''
348 self.use_locked_deps = ''
349 self.use_vendored_sources = ''
351 self.git_version = None
352 self.nix_deps_dir = None
354 def download_stage0(self):
355 """Fetch the build system for Rust, written in Rust
357 This method will build a cache directory, then it will fetch the
358 tarball which has the stage0 compiler used to then bootstrap the Rust
361 Each downloaded tarball is extracted, after that, the script
362 will move all the content to the right place.
364 rustc_channel = self.rustc_channel
365 cargo_channel = self.cargo_channel
366 rustfmt_channel = self.rustfmt_channel
370 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
371 temp_path = temp_file.name
372 with tarfile.open(temp_path, "w:xz"):
375 except tarfile.CompressionError:
378 if self.rustc().startswith(self.bin_root()) and \
379 (not os.path.exists(self.rustc()) or
380 self.program_out_of_date(self.rustc_stamp())):
381 if os.path.exists(self.bin_root()):
382 shutil.rmtree(self.bin_root())
383 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
384 filename = "rust-std-{}-{}{}".format(
385 rustc_channel, self.build, tarball_suffix)
386 pattern = "rust-std-{}".format(self.build)
387 self._download_stage0_helper(filename, pattern, tarball_suffix)
389 filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
391 self._download_stage0_helper(filename, "rustc", tarball_suffix)
392 self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
393 self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
394 lib_dir = "{}/lib".format(self.bin_root())
395 for lib in os.listdir(lib_dir):
396 if lib.endswith(".so"):
397 self.fix_bin_or_dylib("{}/{}".format(lib_dir, lib))
398 with output(self.rustc_stamp()) as rust_stamp:
399 rust_stamp.write(self.date)
401 # This is required so that we don't mix incompatible MinGW
402 # libraries/binaries that are included in rust-std with
403 # the system MinGW ones.
404 if "pc-windows-gnu" in self.build:
405 filename = "rust-mingw-{}-{}{}".format(
406 rustc_channel, self.build, tarball_suffix)
407 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
409 if self.cargo().startswith(self.bin_root()) and \
410 (not os.path.exists(self.cargo()) or
411 self.program_out_of_date(self.cargo_stamp())):
412 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
413 filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
415 self._download_stage0_helper(filename, "cargo", tarball_suffix)
416 self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
417 with output(self.cargo_stamp()) as cargo_stamp:
418 cargo_stamp.write(self.date)
420 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
421 not os.path.exists(self.rustfmt())
422 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
425 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
426 [channel, date] = rustfmt_channel.split('-', 1)
427 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
428 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
429 self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
430 self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
431 with output(self.rustfmt_stamp()) as rustfmt_stamp:
432 rustfmt_stamp.write(self.date + self.rustfmt_channel)
434 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
437 cache_dst = os.path.join(self.build_dir, "cache")
438 rustc_cache = os.path.join(cache_dst, date)
439 if not os.path.exists(rustc_cache):
440 os.makedirs(rustc_cache)
442 url = "{}/dist/{}".format(self._download_url, date)
443 tarball = os.path.join(rustc_cache, filename)
444 if not os.path.exists(tarball):
445 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
446 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
448 def fix_bin_or_dylib(self, fname):
449 """Modifies the interpreter section of 'fname' to fix the dynamic linker,
450 or the RPATH section, to fix the dynamic library search path
452 This method is only required on NixOS and uses the PatchELF utility to
453 change the interpreter/RPATH of ELF executables.
455 Please see https://nixos.org/patchelf.html for more information
457 default_encoding = sys.getdefaultencoding()
459 ostype = subprocess.check_output(
460 ['uname', '-s']).strip().decode(default_encoding)
461 except subprocess.CalledProcessError:
463 except OSError as reason:
464 if getattr(reason, 'winerror', None) is not None:
468 if ostype != "Linux":
471 if not os.path.exists("/etc/NIXOS"):
473 if os.path.exists("/lib"):
476 # At this point we're pretty sure the user is running NixOS
477 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
478 print(nix_os_msg, fname)
480 # Only build `stage0/.nix-deps` once.
481 nix_deps_dir = self.nix_deps_dir
483 nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
484 if not os.path.exists(nix_deps_dir):
485 os.makedirs(nix_deps_dir)
488 # Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
489 "stdenv.cc.bintools",
491 # Needed as a system dependency of `libLLVM-*.so`.
494 # Needed for patching ELF binaries (see doc comment above).
498 # Run `nix-build` to "build" each dependency (which will likely reuse
499 # the existing `/nix/store` copy, or at most download a pre-built copy).
500 # Importantly, we don't rely on `nix-build` printing the `/nix/store`
501 # path on stdout, but use `-o` to symlink it into `stage0/.nix-deps/$dep`,
502 # ensuring garbage collection will never remove the `/nix/store` path
503 # (which would break our patched binaries that hardcode those paths).
506 subprocess.check_output([
507 "nix-build", "<nixpkgs>",
509 "-o", "{}/{}".format(nix_deps_dir, dep),
511 except subprocess.CalledProcessError as reason:
512 print("warning: failed to call nix-build:", reason)
515 self.nix_deps_dir = nix_deps_dir
517 patchelf = "{}/patchelf/bin/patchelf".format(nix_deps_dir)
519 if fname.endswith(".so"):
520 # Dynamic library, patch RPATH to point to system dependencies.
521 dylib_deps = ["zlib"]
523 # Relative default, all binary and dynamic libraries we ship
524 # appear to have this (even when `../lib` is redundant).
526 ] + ["{}/{}/lib".format(nix_deps_dir, dep) for dep in dylib_deps]
527 patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
529 bintools_dir = "{}/stdenv.cc.bintools".format(nix_deps_dir)
530 with open("{}/nix-support/dynamic-linker".format(bintools_dir)) as dynamic_linker:
531 patchelf_args = ["--set-interpreter", dynamic_linker.read().rstrip()]
534 subprocess.check_output([patchelf] + patchelf_args + [fname])
535 except subprocess.CalledProcessError as reason:
536 print("warning: failed to call patchelf:", reason)
539 def rustc_stamp(self):
540 """Return the path for .rustc-stamp
543 >>> rb.build_dir = "build"
544 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
547 return os.path.join(self.bin_root(), '.rustc-stamp')
549 def cargo_stamp(self):
550 """Return the path for .cargo-stamp
553 >>> rb.build_dir = "build"
554 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
557 return os.path.join(self.bin_root(), '.cargo-stamp')
559 def rustfmt_stamp(self):
560 """Return the path for .rustfmt-stamp
563 >>> rb.build_dir = "build"
564 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
567 return os.path.join(self.bin_root(), '.rustfmt-stamp')
569 def program_out_of_date(self, stamp_path, extra=""):
570 """Check if the given program stamp is out of date"""
571 if not os.path.exists(stamp_path) or self.clean:
573 with open(stamp_path, 'r') as stamp:
574 return (self.date + extra) != stamp.read()
577 """Return the binary root directory
580 >>> rb.build_dir = "build"
581 >>> rb.bin_root() == os.path.join("build", "stage0")
584 When the 'build' property is given should be a nested directory:
586 >>> rb.build = "devel"
587 >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
590 return os.path.join(self.build_dir, self.build, "stage0")
592 def get_toml(self, key, section=None):
593 """Returns the value of the given key in config.toml, otherwise returns None
596 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
597 >>> rb.get_toml("key2")
600 If the key does not exists, the result is None:
602 >>> rb.get_toml("key3") is None
605 Optionally also matches the section the key appears in
607 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
608 >>> rb.get_toml('key', 'a')
610 >>> rb.get_toml('key', 'b')
612 >>> rb.get_toml('key', 'c') is None
615 >>> rb.config_toml = 'key1 = true'
616 >>> rb.get_toml("key1")
621 for line in self.config_toml.splitlines():
622 section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
623 if section_match is not None:
624 cur_section = section_match.group(1)
626 match = re.match(r'^{}\s*=(.*)$'.format(key), line)
627 if match is not None:
628 value = match.group(1)
629 if section is None or section == cur_section:
630 return self.get_string(value) or value.strip()
634 """Return config path for cargo"""
635 return self.program_config('cargo')
638 """Return config path for rustc"""
639 return self.program_config('rustc')
642 """Return config path for rustfmt"""
643 if not self.rustfmt_channel:
645 return self.program_config('rustfmt')
647 def program_config(self, program):
648 """Return config path for the given program
651 >>> rb.config_toml = 'rustc = "rustc"\\n'
652 >>> rb.program_config('rustc')
654 >>> rb.config_toml = ''
655 >>> cargo_path = rb.program_config('cargo')
656 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
660 config = self.get_toml(program)
662 return os.path.expanduser(config)
663 return os.path.join(self.bin_root(), "bin", "{}{}".format(
664 program, self.exe_suffix()))
667 def get_string(line):
668 """Return the value between double quotes
670 >>> RustBuild.get_string(' "devel" ')
672 >>> RustBuild.get_string(" 'devel' ")
674 >>> RustBuild.get_string('devel') is None
676 >>> RustBuild.get_string(' "devel ')
679 start = line.find('"')
681 end = start + 1 + line[start + 1:].find('"')
682 return line[start + 1:end]
683 start = line.find('\'')
685 end = start + 1 + line[start + 1:].find('\'')
686 return line[start + 1:end]
691 """Return a suffix for executables"""
692 if sys.platform == 'win32':
696 def bootstrap_binary(self):
697 """Return the path of the bootstrap binary
700 >>> rb.build_dir = "build"
701 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
702 ... "debug", "bootstrap")
705 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
707 def build_bootstrap(self):
708 """Build bootstrap"""
709 build_dir = os.path.join(self.build_dir, "bootstrap")
710 if self.clean and os.path.exists(build_dir):
711 shutil.rmtree(build_dir)
712 env = os.environ.copy()
713 # `CARGO_BUILD_TARGET` breaks bootstrap build.
714 # See also: <https://github.com/rust-lang/rust/issues/70208>.
715 if "CARGO_BUILD_TARGET" in env:
716 del env["CARGO_BUILD_TARGET"]
717 env["RUSTC_BOOTSTRAP"] = '1'
718 env["CARGO_TARGET_DIR"] = build_dir
719 env["RUSTC"] = self.rustc()
720 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
721 (os.pathsep + env["LD_LIBRARY_PATH"]) \
722 if "LD_LIBRARY_PATH" in env else ""
723 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
724 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
725 if "DYLD_LIBRARY_PATH" in env else ""
726 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
727 (os.pathsep + env["LIBRARY_PATH"]) \
728 if "LIBRARY_PATH" in env else ""
729 # preserve existing RUSTFLAGS
730 env.setdefault("RUSTFLAGS", "")
731 env["RUSTFLAGS"] += " -Cdebuginfo=2"
733 build_section = "target.{}".format(self.build_triple())
735 if self.get_toml("crt-static", build_section) == "true":
736 target_features += ["+crt-static"]
737 elif self.get_toml("crt-static", build_section) == "false":
738 target_features += ["-crt-static"]
740 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
741 target_linker = self.get_toml("linker", build_section)
742 if target_linker is not None:
743 env["RUSTFLAGS"] += " -C linker=" + target_linker
744 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
745 if self.get_toml("deny-warnings", "rust") != "false":
746 env["RUSTFLAGS"] += " -Dwarnings"
748 env["PATH"] = os.path.join(self.bin_root(), "bin") + \
749 os.pathsep + env["PATH"]
750 if not os.path.isfile(self.cargo()):
751 raise Exception("no cargo executable found at `{}`".format(
753 args = [self.cargo(), "build", "--manifest-path",
754 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
755 for _ in range(1, self.verbose):
756 args.append("--verbose")
757 if self.use_locked_deps:
758 args.append("--locked")
759 if self.use_vendored_sources:
760 args.append("--frozen")
761 run(args, env=env, verbose=self.verbose)
763 def build_triple(self):
764 """Build triple as in LLVM"""
765 config = self.get_toml('build')
768 return default_build_triple()
770 def check_submodule(self, module, slow_submodules):
771 if not slow_submodules:
772 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
773 cwd=os.path.join(self.rust_root, module),
774 stdout=subprocess.PIPE)
779 def update_submodule(self, module, checked_out, recorded_submodules):
780 module_path = os.path.join(self.rust_root, module)
782 if checked_out is not None:
783 default_encoding = sys.getdefaultencoding()
784 checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
785 if recorded_submodules[module] == checked_out:
788 print("Updating submodule", module)
790 run(["git", "submodule", "-q", "sync", module],
791 cwd=self.rust_root, verbose=self.verbose)
793 update_args = ["git", "submodule", "update", "--init", "--recursive"]
794 if self.git_version >= distutils.version.LooseVersion("2.11.0"):
795 update_args.append("--progress")
796 update_args.append(module)
797 run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True)
799 run(["git", "reset", "-q", "--hard"],
800 cwd=module_path, verbose=self.verbose)
801 run(["git", "clean", "-qdfx"],
802 cwd=module_path, verbose=self.verbose)
804 def update_submodules(self):
805 """Update submodules"""
806 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
807 self.get_toml('submodules') == "false":
810 default_encoding = sys.getdefaultencoding()
812 # check the existence and version of 'git' command
813 git_version_str = require(['git', '--version']).split()[2].decode(default_encoding)
814 self.git_version = distutils.version.LooseVersion(git_version_str)
816 slow_submodules = self.get_toml('fast-submodules') == "false"
819 print('Unconditionally updating all submodules')
821 print('Updating only changed submodules')
822 default_encoding = sys.getdefaultencoding()
823 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
824 ["git", "config", "--file",
825 os.path.join(self.rust_root, ".gitmodules"),
826 "--get-regexp", "path"]
827 ).decode(default_encoding).splitlines()]
828 filtered_submodules = []
829 submodules_names = []
830 for module in submodules:
831 if module.endswith("llvm-project"):
832 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
834 check = self.check_submodule(module, slow_submodules)
835 filtered_submodules.append((module, check))
836 submodules_names.append(module)
837 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
838 cwd=self.rust_root, stdout=subprocess.PIPE)
839 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
840 recorded_submodules = {}
841 for data in recorded:
843 recorded_submodules[data[3]] = data[2]
844 for module in filtered_submodules:
845 self.update_submodule(module[0], module[1], recorded_submodules)
846 print("Submodules updated in %.2f seconds" % (time() - start_time))
848 def set_normal_environment(self):
849 """Set download URL for normal environment"""
850 if 'RUSTUP_DIST_SERVER' in os.environ:
851 self._download_url = os.environ['RUSTUP_DIST_SERVER']
853 self._download_url = 'https://static.rust-lang.org'
855 def set_dev_environment(self):
856 """Set download URL for development environment"""
857 if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
858 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
860 self._download_url = 'https://dev-static.rust-lang.org'
862 def check_vendored_status(self):
863 """Check that vendoring is configured properly"""
864 vendor_dir = os.path.join(self.rust_root, 'vendor')
865 if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
866 if os.environ.get('USER') != os.environ['SUDO_USER']:
867 self.use_vendored_sources = True
868 print('info: looks like you are running this command under `sudo`')
869 print(' and so in order to preserve your $HOME this will now')
870 print(' use vendored sources by default.')
871 if not os.path.exists(vendor_dir):
872 print('error: vendoring required, but vendor directory does not exist.')
873 print(' Run `cargo vendor` without sudo to initialize the '
875 raise Exception("{} not found".format(vendor_dir))
877 if self.use_vendored_sources:
878 if not os.path.exists('.cargo'):
879 os.makedirs('.cargo')
880 with output('.cargo/config') as cargo_config:
882 "[source.crates-io]\n"
883 "replace-with = 'vendored-sources'\n"
884 "registry = 'https://example.com'\n"
886 "[source.vendored-sources]\n"
887 "directory = '{}/vendor'\n"
888 .format(self.rust_root))
890 if os.path.exists('.cargo'):
891 shutil.rmtree('.cargo')
893 def ensure_vendored(self):
894 """Ensure that the vendored sources are available if needed"""
895 vendor_dir = os.path.join(self.rust_root, 'vendor')
896 # Note that this does not handle updating the vendored dependencies if
897 # the rust git repository is updated. Normal development usually does
898 # not use vendoring, so hopefully this isn't too much of a problem.
899 if self.use_vendored_sources and not os.path.exists(vendor_dir):
900 run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
901 verbose=self.verbose, cwd=self.rust_root)
904 def bootstrap(help_triggered):
905 """Configure, fetch, build and run the initial bootstrap"""
907 # If the user is asking for help, let them know that the whole download-and-build
908 # process has to happen before anything is printed out.
910 print("info: Downloading and building bootstrap before processing --help")
911 print(" command. See src/bootstrap/README.md for help with common")
914 parser = argparse.ArgumentParser(description='Build rust')
915 parser.add_argument('--config')
916 parser.add_argument('--build')
917 parser.add_argument('--src')
918 parser.add_argument('--clean', action='store_true')
919 parser.add_argument('-v', '--verbose', action='count', default=0)
921 args = [a for a in sys.argv if a != '-h' and a != '--help']
922 args, _ = parser.parse_known_args(args)
924 # Configure initial bootstrap
926 build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
927 build.verbose = args.verbose
928 build.clean = args.clean
930 # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it
932 toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config
933 if not toml_path and os.path.exists('config.toml'):
934 toml_path = 'config.toml'
937 if not os.path.exists(toml_path):
938 toml_path = os.path.join(build.rust_root, toml_path)
940 with open(toml_path) as config:
941 build.config_toml = config.read()
943 config_verbose = build.get_toml('verbose', 'build')
944 if config_verbose is not None:
945 build.verbose = max(build.verbose, int(config_verbose))
947 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
949 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
951 build.check_vendored_status()
953 build_dir = build.get_toml('build-dir', 'build') or 'build'
954 build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root))
956 data = stage0_data(build.rust_root)
957 build.date = data['date']
958 build.rustc_channel = data['rustc']
959 build.cargo_channel = data['cargo']
961 if "rustfmt" in data:
962 build.rustfmt_channel = data['rustfmt']
965 build.set_dev_environment()
967 build.set_normal_environment()
969 build.update_submodules()
971 # Fetch/build the bootstrap
972 build.build = args.build or build.build_triple()
973 build.download_stage0()
975 build.ensure_vendored()
976 build.build_bootstrap()
980 args = [build.bootstrap_binary()]
981 args.extend(sys.argv[1:])
982 env = os.environ.copy()
983 env["BUILD"] = build.build
984 env["SRC"] = build.rust_root
985 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
986 env["BOOTSTRAP_PYTHON"] = sys.executable
987 env["BUILD_DIR"] = build.build_dir
988 env["RUSTC_BOOTSTRAP"] = '1'
989 env["CARGO"] = build.cargo()
990 env["RUSTC"] = build.rustc()
992 env["BOOTSTRAP_CONFIG"] = toml_path
994 env["RUSTFMT"] = build.rustfmt()
995 run(args, env=env, verbose=build.verbose)
999 """Entry point for the bootstrap process"""
1002 # x.py help <cmd> ...
1003 if len(sys.argv) > 1 and sys.argv[1] == 'help':
1004 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
1007 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
1009 bootstrap(help_triggered)
1010 if not help_triggered:
1011 print("Build completed successfully in {}".format(
1012 format_build_time(time() - start_time)))
1013 except (SystemExit, KeyboardInterrupt) as error:
1014 if hasattr(error, 'code') and isinstance(error.code, int):
1015 exit_code = error.code
1019 if not help_triggered:
1020 print("Build completed unsuccessfully in {}".format(
1021 format_build_time(time() - start_time)))
1025 if __name__ == '__main__':