1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Implementation of rustbuild, the Rust build system.
13 //! This module, and its descendants, are the implementation of the Rust build
14 //! system. Most of this build system is backed by Cargo but the outer layer
15 //! here serves as the ability to orchestrate calling Cargo, sequencing Cargo
16 //! builds, building artifacts like LLVM, etc.
18 //! More documentation can be found in each respective module below.
20 extern crate build_helper;
22 extern crate filetime;
26 extern crate num_cpus;
27 extern crate rustc_serialize;
30 use std::collections::HashMap;
32 use std::fs::{self, File};
33 use std::path::{Component, PathBuf, Path};
34 use std::process::Command;
36 use build_helper::{run_silent, output};
38 use util::{exe, mtime, libdir, add_lib_path};
40 /// A helper macro to `unwrap` a result except also print out details like:
42 /// * The file/line of the panic
43 /// * The expression that failed
44 /// * The error itself
46 /// This is currently used judiciously throughout the build system rather than
47 /// using a `Result` with `try!`, but this may change one day...
49 ($e:expr) => (match $e {
51 Err(e) => panic!("{} failed with {}", stringify!($e), e),
76 pub unsafe fn setup() {}
79 pub use config::Config;
80 pub use flags::{Flags, Subcommand};
82 /// A structure representing a Rust compiler.
84 /// Each compiler has a `stage` that it is associated with and a `host` that
85 /// corresponds to the platform the compiler runs on. This structure is used as
86 /// a parameter to many methods below.
87 #[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)]
88 pub struct Compiler<'a> {
93 /// Global configuration for the build system.
95 /// This structure transitively contains all configuration for the build system.
96 /// All filesystem-encoded configuration is in `config`, all flags are in
97 /// `flags`, and then parsed or probed information is listed in the keys below.
99 /// This structure is a parameter of almost all methods in the build system,
100 /// although most functions are implemented as free functions rather than
101 /// methods specifically on this structure itself (to make it easier to
104 // User-specified configuration via config.toml
107 // User-specified configuration via CLI flags
110 // Derived properties from the above two configurations
116 unstable_features: bool,
117 ver_hash: Option<String>,
118 short_ver_hash: Option<String>,
119 ver_date: Option<String>,
121 package_vers: String,
123 bootstrap_key: String,
124 bootstrap_key_stage0: String,
126 // Probed tools at runtime
127 lldb_version: Option<String>,
128 lldb_python_dir: Option<String>,
130 // Runtime state filled in later on
131 cc: HashMap<String, (gcc::Tool, Option<PathBuf>)>,
132 cxx: HashMap<String, gcc::Tool>,
133 crates: HashMap<String, Crate>,
146 /// The various "modes" of invoking Cargo.
148 /// These entries currently correspond to the various output directories of the
149 /// build system, with each mod generating output in a different directory.
150 #[derive(Clone, Copy)]
152 /// This cargo is going to build the standard library, placing output in the
153 /// "stageN-std" directory.
156 /// This cargo is going to build libtest, placing output in the
157 /// "stageN-test" directory.
160 /// This cargo is going to build librustc and compiler libraries, placing
161 /// output in the "stageN-rustc" directory.
164 /// This cargo is going to some build tool, placing output in the
165 /// "stageN-tools" directory.
170 /// Creates a new set of build configuration from the `flags` on the command
171 /// line and the filesystem `config`.
173 /// By default all build output will be placed in the current directory.
174 pub fn new(flags: Flags, config: Config) -> Build {
175 let cwd = t!(env::current_dir());
176 let src = flags.src.clone().or_else(|| {
177 env::var_os("SRC").map(|x| x.into())
178 }).unwrap_or(cwd.clone());
179 let out = cwd.join("build");
181 let stage0_root = out.join(&config.build).join("stage0/bin");
182 let rustc = match config.rustc {
183 Some(ref s) => PathBuf::from(s),
184 None => stage0_root.join(exe("rustc", &config.build)),
186 let cargo = match config.cargo {
187 Some(ref s) => PathBuf::from(s),
188 None => stage0_root.join(exe("cargo", &config.build)),
190 let local_rebuild = config.local_rebuild;
200 release: String::new(),
201 unstable_features: false,
203 short_ver_hash: None,
205 version: String::new(),
206 local_rebuild: local_rebuild,
207 bootstrap_key: String::new(),
208 bootstrap_key_stage0: String::new(),
209 package_vers: String::new(),
212 crates: HashMap::new(),
214 lldb_python_dir: None,
218 /// Executes the entire build, as configured by the flags and configuration.
219 pub fn build(&mut self) {
224 if let Subcommand::Clean = self.flags.cmd {
225 return clean::clean(self);
228 self.verbose("finding compilers");
230 self.verbose("running sanity check");
232 self.verbose("collecting channel variables");
233 channel::collect(self);
234 // If local-rust is the same major.minor as the current version, then force a local-rebuild
235 let local_version_verbose = output(
236 Command::new(&self.rustc).arg("--version").arg("--verbose"));
237 let local_release = local_version_verbose
238 .lines().filter(|x| x.starts_with("release:"))
239 .next().unwrap().trim_left_matches("release:").trim();
240 if local_release.split('.').take(2).eq(self.release.split('.').take(2)) {
241 self.verbose(&format!("auto-detected local-rebuild {}", local_release));
242 self.local_rebuild = true;
244 self.verbose("updating submodules");
245 self.update_submodules();
246 self.verbose("learning about cargo");
247 metadata::build(self);
252 /// Updates all git submodules that we have.
254 /// This will detect if any submodules are out of date an run the necessary
255 /// commands to sync them all with upstream.
256 fn update_submodules(&self) {
257 struct Submodule<'a> {
263 // The submodule may have staged/unstaged changes
265 // Or could be initialized but never updated
267 // The submodule, itself, has extra commits but those changes haven't been commited to
268 // the (outer) git repository
272 if !self.config.submodules {
275 if fs::metadata(self.src.join(".git")).is_err() {
279 let mut cmd = Command::new("git");
280 cmd.current_dir(&self.src);
283 let git_submodule = || {
284 let mut cmd = Command::new("git");
285 cmd.current_dir(&self.src).arg("submodule");
289 // FIXME: this takes a seriously long time to execute on Windows and a
290 // nontrivial amount of time on Unix, we should have a better way
291 // of detecting whether we need to run all the submodule commands
293 let out = output(git_submodule().arg("status"));
294 let mut submodules = vec![];
295 for line in out.lines() {
296 // NOTE `git submodule status` output looks like this:
298 // -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
299 // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
300 // e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
302 // The first character can be '-', '+' or ' ' and denotes the `State` of the submodule
303 // Right next to this character is the SHA-1 of the submodule HEAD
304 // And after that comes the path to the submodule
305 let path = Path::new(line[1..].split(' ').skip(1).next().unwrap());
306 let state = if line.starts_with('-') {
307 State::NotInitialized
308 } else if line.starts_with('+') {
310 } else if line.starts_with(' ') {
313 panic!("unexpected git submodule state: {:?}", line.chars().next());
316 submodules.push(Submodule { path: path, state: state })
319 self.run(git_submodule().arg("sync"));
321 for submodule in submodules {
322 // If using llvm-root then don't touch the llvm submodule.
323 if submodule.path.components().any(|c| c == Component::Normal("llvm".as_ref())) &&
324 self.config.target_config.get(&self.config.build)
325 .and_then(|c| c.llvm_config.as_ref()).is_some()
330 if submodule.path.components().any(|c| c == Component::Normal("jemalloc".as_ref())) &&
331 !self.config.use_jemalloc
336 // `submodule.path` is the relative path to a submodule (from the repository root)
337 // `submodule_path` is the path to a submodule from the cwd
339 // use `submodule.path` when e.g. executing a submodule specific command from the
341 // use `submodule_path` when e.g. executing a normal git command for the submodule
342 // (set via `current_dir`)
343 let submodule_path = self.src.join(submodule.path);
345 match submodule.state {
346 State::MaybeDirty => {
347 // drop staged changes
348 self.run(git().current_dir(&submodule_path)
349 .args(&["reset", "--hard"]));
350 // drops unstaged changes
351 self.run(git().current_dir(&submodule_path)
352 .args(&["clean", "-fdx"]));
354 State::NotInitialized => {
355 self.run(git_submodule().arg("init").arg(submodule.path));
356 self.run(git_submodule().arg("update").arg(submodule.path));
358 State::OutOfSync => {
359 // drops submodule commits that weren't reported to the (outer) git repository
360 self.run(git_submodule().arg("update").arg(submodule.path));
361 self.run(git().current_dir(&submodule_path)
362 .args(&["reset", "--hard"]));
363 self.run(git().current_dir(&submodule_path)
364 .args(&["clean", "-fdx"]));
370 /// Clear out `dir` if `input` is newer.
372 /// After this executes, it will also ensure that `dir` exists.
373 fn clear_if_dirty(&self, dir: &Path, input: &Path) {
374 let stamp = dir.join(".stamp");
375 if mtime(&stamp) < mtime(input) {
376 self.verbose(&format!("Dirty - {}", dir.display()));
377 let _ = fs::remove_dir_all(dir);
378 } else if stamp.exists() {
381 t!(fs::create_dir_all(dir));
382 t!(File::create(stamp));
385 /// Prepares an invocation of `cargo` to be run.
387 /// This will create a `Command` that represents a pending execution of
388 /// Cargo. This cargo will be configured to use `compiler` as the actual
389 /// rustc compiler, its output will be scoped by `mode`'s output directory,
390 /// it will pass the `--target` flag for the specified `target`, and will be
391 /// executing the Cargo command `cmd`.
396 cmd: &str) -> Command {
397 let mut cargo = Command::new(&self.cargo);
398 let out_dir = self.stage_out(compiler, mode);
399 cargo.env("CARGO_TARGET_DIR", out_dir)
401 .arg("-j").arg(self.jobs().to_string())
402 .arg("--target").arg(target);
404 // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005
405 // Force cargo to output binaries with disambiguating hashes in the name
406 cargo.env("__CARGO_DEFAULT_LIB_METADATA", "1");
409 if compiler.stage == 0 && self.local_rebuild {
410 // Assume the local-rebuild rustc already has stage1 features.
413 stage = compiler.stage;
416 // Customize the compiler we're running. Specify the compiler to cargo
417 // as our shim and then pass it some various options used to configure
418 // how the actual compiler itself is called.
420 // These variables are primarily all read by
421 // src/bootstrap/{rustc,rustdoc.rs}
422 cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc"))
423 .env("RUSTC_REAL", self.compiler_path(compiler))
424 .env("RUSTC_STAGE", stage.to_string())
425 .env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
426 .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string())
427 .env("RUSTC_CODEGEN_UNITS",
428 self.config.rust_codegen_units.to_string())
429 .env("RUSTC_DEBUG_ASSERTIONS",
430 self.config.rust_debug_assertions.to_string())
431 .env("RUSTC_SNAPSHOT", &self.rustc)
432 .env("RUSTC_SYSROOT", self.sysroot(compiler))
433 .env("RUSTC_LIBDIR", self.rustc_libdir(compiler))
434 .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir())
435 .env("RUSTC_RPATH", self.config.rust_rpath.to_string())
436 .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
437 .env("RUSTDOC_REAL", self.rustdoc(compiler))
438 .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
440 self.add_bootstrap_key(&mut cargo);
442 // Specify some various options for build scripts used throughout
445 // FIXME: the guard against msvc shouldn't need to be here
446 if !target.contains("msvc") {
447 cargo.env(format!("CC_{}", target), self.cc(target))
448 .env(format!("AR_{}", target), self.ar(target).unwrap()) // only msvc is None
449 .env(format!("CFLAGS_{}", target), self.cflags(target).join(" "));
452 // Environment variables *required* needed throughout the build
454 // FIXME: should update code to not require this env var
455 cargo.env("CFG_COMPILER_HOST_TRIPLE", target);
457 if self.config.verbose || self.flags.verbose {
460 if self.config.rust_optimize {
461 cargo.arg("--release");
466 /// Get a path to the compiler specified.
467 fn compiler_path(&self, compiler: &Compiler) -> PathBuf {
468 if compiler.is_snapshot(self) {
471 self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host))
475 /// Get the specified tool built by the specified compiler
476 fn tool(&self, compiler: &Compiler, tool: &str) -> PathBuf {
477 self.cargo_out(compiler, Mode::Tool, compiler.host)
478 .join(exe(tool, compiler.host))
481 /// Get the `rustdoc` executable next to the specified compiler
482 fn rustdoc(&self, compiler: &Compiler) -> PathBuf {
483 let mut rustdoc = self.compiler_path(compiler);
485 rustdoc.push(exe("rustdoc", compiler.host));
489 /// Get a `Command` which is ready to run `tool` in `stage` built for
491 fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command {
492 let mut cmd = Command::new(self.tool(&compiler, tool));
493 let host = compiler.host;
495 self.cargo_out(compiler, Mode::Libstd, host).join("deps"),
496 self.cargo_out(compiler, Mode::Libtest, host).join("deps"),
497 self.cargo_out(compiler, Mode::Librustc, host).join("deps"),
498 self.cargo_out(compiler, Mode::Tool, host).join("deps"),
500 add_lib_path(paths, &mut cmd);
504 /// Get the space-separated set of activated features for the standard
506 fn std_features(&self) -> String {
507 let mut features = String::new();
508 if self.config.debug_jemalloc {
509 features.push_str(" debug-jemalloc");
511 if self.config.use_jemalloc {
512 features.push_str(" jemalloc");
514 if self.config.backtrace {
515 features.push_str(" backtrace");
520 /// Get the space-separated set of activated features for the compiler.
521 fn rustc_features(&self) -> String {
522 let mut features = String::new();
523 if self.config.use_jemalloc {
524 features.push_str(" jemalloc");
529 /// Component directory that Cargo will produce output into (e.g.
531 fn cargo_dir(&self) -> &'static str {
532 if self.config.rust_optimize {"release"} else {"debug"}
535 /// Returns the sysroot for the `compiler` specified that *this build system
538 /// That is, the sysroot for the stage0 compiler is not what the compiler
539 /// thinks it is by default, but it's the same as the default for stages
541 fn sysroot(&self, compiler: &Compiler) -> PathBuf {
542 if compiler.stage == 0 {
543 self.out.join(compiler.host).join("stage0-sysroot")
545 self.out.join(compiler.host).join(format!("stage{}", compiler.stage))
549 /// Returns the libdir where the standard library and other artifacts are
550 /// found for a compiler's sysroot.
551 fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf {
552 self.sysroot(compiler).join("lib").join("rustlib")
553 .join(target).join("lib")
556 /// Returns the root directory for all output generated in a particular
557 /// stage when running with a particular host compiler.
559 /// The mode indicates what the root directory is for.
560 fn stage_out(&self, compiler: &Compiler, mode: Mode) -> PathBuf {
561 let suffix = match mode {
562 Mode::Libstd => "-std",
563 Mode::Libtest => "-test",
564 Mode::Tool => "-tools",
565 Mode::Librustc => "-rustc",
567 self.out.join(compiler.host)
568 .join(format!("stage{}{}", compiler.stage, suffix))
571 /// Returns the root output directory for all Cargo output in a given stage,
572 /// running a particular comipler, wehther or not we're building the
573 /// standard library, and targeting the specified architecture.
577 target: &str) -> PathBuf {
578 self.stage_out(compiler, mode).join(target).join(self.cargo_dir())
581 /// Root output directory for LLVM compiled for `target`
583 /// Note that if LLVM is configured externally then the directory returned
584 /// will likely be empty.
585 fn llvm_out(&self, target: &str) -> PathBuf {
586 self.out.join(target).join("llvm")
589 /// Output directory for all documentation for a target
590 fn doc_out(&self, target: &str) -> PathBuf {
591 self.out.join(target).join("doc")
594 /// Returns true if no custom `llvm-config` is set for the specified target.
596 /// If no custom `llvm-config` was specified then Rust's llvm will be used.
597 fn is_rust_llvm(&self, target: &str) -> bool {
598 match self.config.target_config.get(target) {
599 Some(ref c) => c.llvm_config.is_none(),
604 /// Returns the path to `llvm-config` for the specified target.
606 /// If a custom `llvm-config` was specified for target then that's returned
608 fn llvm_config(&self, target: &str) -> PathBuf {
609 let target_config = self.config.target_config.get(target);
610 if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
613 self.llvm_out(&self.config.build).join("bin")
614 .join(exe("llvm-config", target))
618 /// Returns the path to `FileCheck` binary for the specified target
619 fn llvm_filecheck(&self, target: &str) -> PathBuf {
620 let target_config = self.config.target_config.get(target);
621 if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
622 s.parent().unwrap().join(exe("FileCheck", target))
624 let base = self.llvm_out(&self.config.build).join("build");
625 let exe = exe("FileCheck", target);
626 if self.config.build.contains("msvc") {
627 base.join("Release/bin").join(exe)
629 base.join("bin").join(exe)
634 /// Root output directory for rust_test_helpers library compiled for
636 fn test_helpers_out(&self, target: &str) -> PathBuf {
637 self.out.join(target).join("rust-test-helpers")
640 /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic
641 /// library lookup path.
642 fn add_rustc_lib_path(&self, compiler: &Compiler, cmd: &mut Command) {
643 // Windows doesn't need dylib path munging because the dlls for the
644 // compiler live next to the compiler and the system will find them
650 add_lib_path(vec![self.rustc_libdir(compiler)], cmd);
653 /// Adds the compiler's bootstrap key to the environment of `cmd`.
654 fn add_bootstrap_key(&self, cmd: &mut Command) {
655 cmd.env("RUSTC_BOOTSTRAP", "1");
656 // FIXME: Transitionary measure to bootstrap using the old bootstrap logic.
657 // Remove this once the bootstrap compiler uses the new login in Issue #36548.
658 cmd.env("RUSTC_BOOTSTRAP_KEY", "62b3e239");
661 /// Returns the compiler's libdir where it stores the dynamic libraries that
662 /// it itself links against.
664 /// For example this returns `<sysroot>/lib` on Unix and `<sysroot>/bin` on
666 fn rustc_libdir(&self, compiler: &Compiler) -> PathBuf {
667 if compiler.is_snapshot(self) {
668 self.rustc_snapshot_libdir()
670 self.sysroot(compiler).join(libdir(compiler.host))
674 /// Returns the libdir of the snapshot compiler.
675 fn rustc_snapshot_libdir(&self) -> PathBuf {
676 self.rustc.parent().unwrap().parent().unwrap()
677 .join(libdir(&self.config.build))
680 /// Runs a command, printing out nice contextual information if it fails.
681 fn run(&self, cmd: &mut Command) {
682 self.verbose(&format!("running: {:?}", cmd));
686 /// Prints a message if this build is configured in verbose mode.
687 fn verbose(&self, msg: &str) {
688 if self.flags.verbose || self.config.verbose {
693 /// Returns the number of parallel jobs that have been configured for this
695 fn jobs(&self) -> u32 {
696 self.flags.jobs.unwrap_or(num_cpus::get() as u32)
699 /// Returns the path to the C compiler for the target specified.
700 fn cc(&self, target: &str) -> &Path {
701 self.cc[target].0.path()
704 /// Returns a list of flags to pass to the C compiler for the target
706 fn cflags(&self, target: &str) -> Vec<String> {
707 // Filter out -O and /O (the optimization flags) that we picked up from
708 // gcc-rs because the build scripts will determine that for themselves.
709 let mut base = self.cc[target].0.args().iter()
710 .map(|s| s.to_string_lossy().into_owned())
711 .filter(|s| !s.starts_with("-O") && !s.starts_with("/O"))
712 .collect::<Vec<_>>();
714 // If we're compiling on OSX then we add a few unconditional flags
715 // indicating that we want libc++ (more filled out than libstdc++) and
716 // we want to compile for 10.7. This way we can ensure that
717 // LLVM/jemalloc/etc are all properly compiled.
718 if target.contains("apple-darwin") {
719 base.push("-stdlib=libc++".into());
721 // This is a hack, because newer binutils broke things on some vms/distros
722 // (i.e., linking against unknown relocs disabled by the following flag)
723 // See: https://github.com/rust-lang/rust/issues/34978
725 "i586-unknown-linux-gnu" |
726 "i686-unknown-linux-musl" |
727 "x86_64-unknown-linux-musl" => {
728 base.push("-Wa,-mrelax-relocations=no".into());
735 /// Returns the path to the `ar` archive utility for the target specified.
736 fn ar(&self, target: &str) -> Option<&Path> {
737 self.cc[target].1.as_ref().map(|p| &**p)
740 /// Returns the path to the C++ compiler for the target specified, may panic
741 /// if no C++ compiler was configured for the target.
742 fn cxx(&self, target: &str) -> &Path {
743 match self.cxx.get(target) {
745 None => panic!("\n\ntarget `{}` is not configured as a host,
746 only as a target\n\n", target),
750 /// Returns flags to pass to the compiler to generate code for `target`.
751 fn rustc_flags(&self, target: &str) -> Vec<String> {
752 // New flags should be added here with great caution!
754 // It's quite unfortunate to **require** flags to generate code for a
755 // target, so it should only be passed here if absolutely necessary!
756 // Most default configuration should be done through target specs rather
757 // than an entry here.
759 let mut base = Vec::new();
760 if target != self.config.build && !target.contains("msvc") &&
761 !target.contains("emscripten") {
762 base.push(format!("-Clinker={}", self.cc(target).display()));
767 /// Returns the "musl root" for this `target`, if defined
768 fn musl_root(&self, target: &str) -> Option<&Path> {
769 self.config.target_config.get(target)
770 .and_then(|t| t.musl_root.as_ref())
771 .or(self.config.musl_root.as_ref())
776 impl<'a> Compiler<'a> {
777 /// Creates a new complier for the specified stage/host
778 fn new(stage: u32, host: &'a str) -> Compiler<'a> {
779 Compiler { stage: stage, host: host }
782 /// Returns whether this is a snapshot compiler for `build`'s configuration
783 fn is_snapshot(&self, build: &Build) -> bool {
784 self.stage == 0 && self.host == build.config.build