From ea7fef1ccf4b44effbc3bbc902f9b0b9dbd251a4 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 17 Dec 2018 21:10:59 -0800 Subject: [PATCH] bootstrap: Link LLVM as a dylib with ThinLTO When building a distributed compiler on Linux where we use ThinLTO to create the LLVM shared object this commit switches the compiler to dynamically linking that LLVM artifact instead of statically linking to LLVM. The primary goal here is to reduce CI compile times, avoiding two+ ThinLTO builds of all of LLVM. By linking dynamically to LLVM we'll reuse the one ThinLTO step done by LLVM's build itself. Lots of discussion about this change can be found [here] and down. A perf run will show whether this is worth it or not! [here]: https://github.com/rust-lang/rust/pull/53245#issuecomment-417015334 --- src/bootstrap/check.rs | 5 ---- src/bootstrap/compile.rs | 49 +++++++------------------------------- src/bootstrap/dist.rs | 8 +++---- src/bootstrap/tool.rs | 2 +- src/librustc_llvm/build.rs | 4 ++++ 5 files changed, 17 insertions(+), 51 deletions(-) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 050db936daa..cc539d4c895 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -38,7 +38,6 @@ fn run(self, builder: &Builder) { builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target)); run_cargo(builder, &mut cargo, - vec![], &libstd_stamp(builder, compiler, target), true); @@ -85,7 +84,6 @@ fn run(self, builder: &Builder) { builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target)); run_cargo(builder, &mut cargo, - vec![], &librustc_stamp(builder, compiler, target), true); @@ -136,7 +134,6 @@ fn run(self, builder: &Builder) { let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage)); run_cargo(builder, &mut cargo, - vec![], &codegen_backend_stamp(builder, compiler, target, backend), true); } @@ -174,7 +171,6 @@ fn run(self, builder: &Builder) { builder.info(&format!("Checking test artifacts ({} -> {})", &compiler.host, target)); run_cargo(builder, &mut cargo, - vec![], &libtest_stamp(builder, compiler, target), true); @@ -222,7 +218,6 @@ fn run(self, builder: &Builder) { println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target); run_cargo(builder, &mut cargo, - vec![], &rustdoc_stamp(builder, compiler, target), true); diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 821bd002e95..0d2546a0e9c 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -19,6 +19,7 @@ use filetime::FileTime; use serde_json; +use crate::dist; use crate::util::{exe, libdir, is_dylib}; use crate::{Compiler, Mode, GitRepo}; use crate::native; @@ -104,7 +105,6 @@ fn run(self, builder: &Builder) { &compiler.host, target)); run_cargo(builder, &mut cargo, - vec![], &libstd_stamp(builder, compiler, target), false); @@ -365,7 +365,6 @@ fn run(self, builder: &Builder) { &compiler.host, target)); run_cargo(builder, &mut cargo, - vec![], &libtest_stamp(builder, compiler, target), false); @@ -493,7 +492,6 @@ fn run(self, builder: &Builder) { compiler.stage, &compiler.host, target)); run_cargo(builder, &mut cargo, - vec![], &librustc_stamp(builder, compiler, target), false); @@ -636,47 +634,18 @@ fn run(self, builder: &Builder) { let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); - let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc"); + let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build"); cargo.arg("--manifest-path") .arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml")); rustc_cargo_env(builder, &mut cargo); let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend); - let mut cargo_tails_args = vec![]; - - if builder.config.llvm_thin_lto { - cargo_tails_args.push("--".to_string()); - - let num_jobs = builder.jobs(); - - if !target.contains("msvc") { - // Here we assume that the linker is clang. If it's not, there'll - // be linker errors. - cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string()); - cargo_tails_args.push("-Clink-arg=-flto=thin".to_string()); - - if builder.config.llvm_optimize { - cargo_tails_args.push("-Clink-arg=-O2".to_string()); - } - - // Let's make LLD respect the `-j` option. - let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs); - cargo_tails_args.push(num_jobs_arg); - } else { - // Here we assume that the linker is lld-link.exe. lld-link.exe - // does not need the extra arguments except for num_jobs - let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs); - cargo_tails_args.push(num_jobs_arg); - } - } - let tmp_stamp = out_dir.join(".tmp.stamp"); let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage)); let files = run_cargo(builder, cargo.arg("--features").arg(features), - cargo_tails_args, &tmp_stamp, false); if builder.config.dry_run { @@ -749,7 +718,9 @@ pub fn build_codegen_backend(builder: &Builder, "libstdc++.a"); cargo.env("LLVM_STATIC_STDCPP", file); } - if builder.config.llvm_link_shared { + if builder.config.llvm_link_shared || + (builder.config.llvm_thin_lto && backend != "emscripten") + { cargo.env("LLVM_LINK_SHARED", "1"); } } @@ -989,6 +960,8 @@ fn run(self, builder: &Builder) -> Compiler { copy_lld_to_sysroot(builder, target_compiler, &lld_install); } + dist::maybe_install_llvm_dylib(builder, target_compiler.host, &sysroot); + // Link the compiler binary itself into place let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); let rustc = out_dir.join(exe("rustc_binary", &*host)); @@ -1015,7 +988,6 @@ pub fn add_to_sysroot(builder: &Builder, sysroot_dst: &Path, stamp: &Path) { pub fn run_cargo(builder: &Builder, cargo: &mut Command, - tail_args: Vec, stamp: &Path, is_check: bool) -> Vec @@ -1038,7 +1010,7 @@ pub fn run_cargo(builder: &Builder, // files we need to probe for later. let mut deps = Vec::new(); let mut toplevel = Vec::new(); - let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { + let ok = stream_cargo(builder, cargo, &mut |msg| { let filenames = match msg { CargoMessage::CompilerArtifact { filenames, .. } => filenames, _ => return, @@ -1163,7 +1135,6 @@ pub fn run_cargo(builder: &Builder, pub fn stream_cargo( builder: &Builder, cargo: &mut Command, - tail_args: Vec, cb: &mut dyn FnMut(CargoMessage), ) -> bool { if builder.config.dry_run { @@ -1174,10 +1145,6 @@ pub fn stream_cargo( cargo.arg("--message-format").arg("json") .stdout(Stdio::piped()); - for arg in tail_args { - cargo.arg(arg); - } - builder.verbose(&format!("running: {:?}", cargo)); let mut child = match cargo.spawn() { Ok(child) => child, diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9f97e57b456..97a359639cb 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1877,13 +1877,13 @@ fn run(self, builder: &Builder) { // LLVM tools are linked dynamically. // Note: This function does no yet support Windows but we also don't support // linking LLVM tools dynamically on Windows yet. -fn maybe_install_llvm_dylib(builder: &Builder, - target: Interned, - image: &Path) { +pub fn maybe_install_llvm_dylib(builder: &Builder, + target: Interned, + sysroot: &Path) { let src_libdir = builder .llvm_out(target) .join("lib"); - let dst_libdir = image.join("lib/rustlib").join(&*target).join("lib"); + let dst_libdir = sysroot.join("lib/rustlib").join(&*target).join("lib"); t!(fs::create_dir_all(&dst_libdir)); if target.contains("apple-darwin") { diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 7782351a552..d31ea0f8458 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -77,7 +77,7 @@ fn run(self, builder: &Builder) -> Option { let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool)); builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target)); let mut duplicates = Vec::new(); - let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| { + let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| { // Only care about big things like the RLS/Cargo for now match tool { | "rls" diff --git a/src/librustc_llvm/build.rs b/src/librustc_llvm/build.rs index 238e79dfd22..98cf20a7ba7 100644 --- a/src/librustc_llvm/build.rs +++ b/src/librustc_llvm/build.rs @@ -132,6 +132,10 @@ fn main() { continue; } + if flag.starts_with("-flto") { + continue; + } + // -Wdate-time is not supported by the netbsd cross compiler if is_crossed && target.contains("netbsd") && flag.contains("date-time") { continue; -- 2.44.0