rustbuild: Fix dist for non-host targets

The `rust-std` package that we produce is expected to have not only the standard
library but also libtest for compiling unit tests. Unfortunately this does not
currently happen due to the way rustbuild is structured.

There are currently two main stages of compilation in rustbuild, one for the
standard library and one for the compiler. This is primarily done to allow us to
fill in the sysroot right after the standard library has finished compiling to
continue compiling the rest of the crates. Consequently the entire compiler does
not have to explicitly depend on the standard library, and this also should
allow us to pull in crates.io dependencies into the build in the future because
they'll just naturally build against the std we just produced.

These phases, however, do not represent a cross-compiled build. Target-only
builds also require libtest, and libtest is currently part of the
all-encompassing "compiler build". There's unfortunately no way to learn about
just libtest and its dependencies (in a great and robust fashion) so to ensure
that we can copy the right artifacts over this commit introduces a new build
step, libtest.

The new libtest build step has documentation, dist, and link steps as std/rustc
already do. The compiler now depends on libtest instead of libstd, and all
compiler crates can now assume that test and its dependencies are implicitly
part of the sysroot (hence explicit dependencies being removed). This makes the
build a tad less parallel as in theory many rustc crates can be compiled in
parallel with libtest, but this likely isn't where we really need parallelism
either (all the time is still spent in the compiler).

All in all this allows the `dist-std` step to depend on both libstd and libtest,
so `rust-std` packages produced by rustbuild should start having both the
standard library and libtest.

Closes #32523

diff --git a/src/bootstrap/build/compile.rs b/src/bootstrap/build/compile.rs
index 95555aa3796889143df1ef07f87ecec4172064a2..dee586c76993668e09797cb0a90fc79f1dcfe066 100644 (file)
--- a/src/bootstrap/build/compile.rs
+++ b/src/bootstrap/build/compile.rs
@@ -123,6 +123,38 @@ fn build_startup_objects(build: &Build, target: &str, into: &Path) {
     }
 }
 
+/// Build libtest.
+///
+/// This will build libtest and supporting libraries for a particular stage of
+/// the build using the `compiler` targeting the `target` architecture. The
+/// artifacts created will also be linked into the sysroot directory.
+pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
+    println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
+             compiler.host, target);
+    let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
+    build.clear_if_dirty(&out_dir, &libstd_shim(build, compiler, target));
+    let mut cargo = build.cargo(compiler, Mode::Libtest, target, "build");
+    cargo.arg("--manifest-path")
+         .arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
+    build.run(&mut cargo);
+    test_link(build, target, compiler, compiler.host);
+}
+
+/// Link all libtest rlibs/dylibs into the sysroot location.
+///
+/// Links those artifacts generated in the given `stage` for `target` produced
+/// by `compiler` into `host`'s sysroot.
+pub fn test_link(build: &Build,
+                 target: &str,
+                 compiler: &Compiler,
+                 host: &str) {
+    let target_compiler = Compiler::new(compiler.stage, host);
+    let libdir = build.sysroot_libdir(&target_compiler, target);
+    let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
+    add_to_sysroot(&out_dir, &libdir);
+}
+
+
 /// Build the compiler.
 ///
 /// This will build the compiler for a particular stage of the build using
@@ -133,7 +165,7 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
              compiler.stage, compiler.host, target);
 
     let out_dir = build.cargo_out(compiler, Mode::Librustc, target);
-    build.clear_if_dirty(&out_dir, &libstd_shim(build, compiler, target));
+    build.clear_if_dirty(&out_dir, &libtest_shim(build, compiler, target));
 
     let mut cargo = build.cargo(compiler, Mode::Librustc, target, "build");
     cargo.arg("--features").arg(build.rustc_features())
@@ -202,6 +234,12 @@ fn libstd_shim(build: &Build, compiler: &Compiler, target: &str) -> PathBuf {
     build.cargo_out(compiler, Mode::Libstd, target).join("libstd_shim.rlib")
 }
 
+/// Cargo's output path for libtest in a given stage, compiled by a particular
+/// compiler for the specified target.
+fn libtest_shim(build: &Build, compiler: &Compiler, target: &str) -> PathBuf {
+    build.cargo_out(compiler, Mode::Libtest, target).join("libtest_shim.rlib")
+}
+
 fn compiler_file(compiler: &Path, file: &str) -> String {
     output(Command::new(compiler)
                    .arg(format!("-print-file-name={}", file))).trim().to_string()

diff --git a/src/bootstrap/build/doc.rs b/src/bootstrap/build/doc.rs
index d8b02dce222cd0312eb3f9d1469cf2cf3d6344f0..50c0c56807bc205361a5f3724179b341e4ae998d 100644 (file)
--- a/src/bootstrap/build/doc.rs
+++ b/src/bootstrap/build/doc.rs
@@ -122,6 +122,22 @@ pub fn std(build: &Build, stage: u32, host: &str, out: &Path) {
     cp_r(&out_dir, out)
 }
 
+pub fn test(build: &Build, stage: u32, host: &str, out: &Path) {
+    println!("Documenting stage{} test ({})", stage, host);
+    let compiler = Compiler::new(stage, host);
+    let out_dir = build.stage_out(&compiler, Mode::Libtest)
+                       .join(host).join("doc");
+    let rustdoc = build.rustdoc(&compiler);
+
+    build.clear_if_dirty(&out_dir, &rustdoc);
+
+    let mut cargo = build.cargo(&compiler, Mode::Libtest, host, "doc");
+    cargo.arg("--manifest-path")
+         .arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
+    build.run(&mut cargo);
+    cp_r(&out_dir, out)
+}
+
 pub fn rustc(build: &Build, stage: u32, host: &str, out: &Path) {
     println!("Documenting stage{} compiler ({})", stage, host);
     let compiler = Compiler::new(stage, host);

diff --git a/src/bootstrap/build/mod.rs b/src/bootstrap/build/mod.rs
index 3427feeca6e2500f83d1c000bf3b4185263a009c..bd217940bfbb2151608e22f2dd3cbc1a684f9b86 100644 (file)
--- a/src/bootstrap/build/mod.rs
+++ b/src/bootstrap/build/mod.rs
@@ -88,6 +88,7 @@ pub struct Build {
 
 pub enum Mode {
     Libstd,
+    Libtest,
     Librustc,
     Tool,
 }
@@ -141,9 +142,13 @@ pub fn build(&mut self) {
             return clean::clean(self);
         }
 
+        self.verbose("finding compilers");
         cc::find(self);
+        self.verbose("running sanity check");
         sanity::check(self);
+        self.verbose("collecting channel variables");
         channel::collect(self);
+        self.verbose("updating submodules");
         self.update_submodules();
 
         for target in step::all(self) {
@@ -158,12 +163,18 @@ pub fn build(&mut self) {
                 Libstd { compiler } => {
                     compile::std(self, target.target, &compiler);
                 }
+                Libtest { compiler } => {
+                    compile::test(self, target.target, &compiler);
+                }
                 Librustc { compiler } => {
                     compile::rustc(self, target.target, &compiler);
                 }
                 LibstdLink { compiler, host } => {
                     compile::std_link(self, target.target, &compiler, host);
                 }
+                LibtestLink { compiler, host } => {
+                    compile::test_link(self, target.target, &compiler, host);
+                }
                 LibrustcLink { compiler, host } => {
                     compile::rustc_link(self, target.target, &compiler, host);
                 }
@@ -203,6 +214,9 @@ pub fn build(&mut self) {
                 DocStd { stage } => {
                     doc::std(self, stage, target.target, &doc_out);
                 }
+                DocTest { stage } => {
+                    doc::test(self, stage, target.target, &doc_out);
+                }
                 DocRustc { stage } => {
                     doc::rustc(self, stage, target.target, &doc_out);
                 }
@@ -360,6 +374,7 @@ fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command {
         let host = compiler.host;
         let paths = vec![
             self.cargo_out(compiler, Mode::Libstd, host).join("deps"),
+            self.cargo_out(compiler, Mode::Libtest, host).join("deps"),
             self.cargo_out(compiler, Mode::Librustc, host).join("deps"),
         ];
         add_lib_path(paths, &mut cmd);
@@ -414,7 +429,8 @@ fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf {
     fn stage_out(&self, compiler: &Compiler, mode: Mode) -> PathBuf {
         let suffix = match mode {
             Mode::Libstd => "-std",
-            _ => "-rustc",
+            Mode::Libtest => "-test",
+            Mode::Tool | Mode::Librustc => "-rustc",
         };
         self.out.join(compiler.host)
                 .join(format!("stage{}{}", compiler.stage, suffix))

diff --git a/src/bootstrap/build/step.rs b/src/bootstrap/build/step.rs
index 107b0688b688920a712ba5c60db527516be30965..4e3aacd3720ff7d3c607784e91dba324665068e7 100644 (file)
--- a/src/bootstrap/build/step.rs
+++ b/src/bootstrap/build/step.rs
@@ -25,19 +25,22 @@ macro_rules! targets {
             // compiler executable itself, not any of the support libraries
             (rustc, Rustc { stage: u32 }),
 
-            // Steps for the two main cargo builds, one for the standard library
-            // and one for the compiler itself. These are parameterized over the
-            // stage output they're going to be placed in along with the
-            // compiler which is producing the copy of libstd or librustc
+            // Steps for the two main cargo builds. These are parameterized over
+            // the compiler which is producing the artifact.
             (libstd, Libstd { compiler: Compiler<'a> }),
+            (libtest, Libtest { compiler: Compiler<'a> }),
             (librustc, Librustc { compiler: Compiler<'a> }),
 
-            // Links the standard library/librustc produced by the compiler
-            // provided into the host's directory also provided.
+            // Links the target produced by the compiler provided into the
+            // host's directory also provided.
             (libstd_link, LibstdLink {
                 compiler: Compiler<'a>,
                 host: &'a str
             }),
+            (libtest_link, LibtestLink {
+                compiler: Compiler<'a>,
+                host: &'a str
+            }),
             (librustc_link, LibrustcLink {
                 compiler: Compiler<'a>,
                 host: &'a str
@@ -67,6 +70,7 @@ macro_rules! targets {
             (doc_style, DocStyle { stage: u32 }),
             (doc_standalone, DocStandalone { stage: u32 }),
             (doc_std, DocStd { stage: u32 }),
+            (doc_test, DocTest { stage: u32 }),
             (doc_rustc, DocRustc { stage: u32 }),
             (doc_error_index, DocErrorIndex { stage: u32 }),
 
@@ -162,10 +166,10 @@ fn top_level(build: &Build) -> Vec<Step> {
 
                 if host.target == build.config.build {
                     targets.push(host.target(target)
-                                     .libstd(host.compiler(stage)));
+                                     .libtest(host.compiler(stage)));
                 } else {
                     targets.push(host.target(target)
-                                     .libstd_link(t.compiler(stage), host.target));
+                                     .libtest_link(t.compiler(stage), host.target));
                 }
             }
         }
@@ -246,7 +250,10 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
                 vec![self.librustc(compiler)]
             }
             Source::Librustc { compiler } => {
-                vec![self.libstd(compiler), self.llvm(())]
+                vec![self.libtest(compiler), self.llvm(())]
+            }
+            Source::Libtest { compiler } => {
+                vec![self.libstd(compiler)]
             }
             Source::Libstd { compiler } => {
                 vec![self.compiler_rt(()),
@@ -254,7 +261,10 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
             }
             Source::LibrustcLink { compiler, host } => {
                 vec![self.librustc(compiler),
-                     self.libstd_link(compiler, host)]
+                     self.libtest_link(compiler, host)]
+            }
+            Source::LibtestLink { compiler, host } => {
+                vec![self.libtest(compiler), self.libstd_link(compiler, host)]
             }
             Source::LibstdLink { compiler, host } => {
                 vec![self.libstd(compiler),
@@ -267,6 +277,9 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
             Source::DocStd { stage } => {
                 vec![self.libstd(self.compiler(stage))]
             }
+            Source::DocTest { stage } => {
+                vec![self.libtest(self.compiler(stage))]
+            }
             Source::DocBook { stage } |
             Source::DocNomicon { stage } |
             Source::DocStyle { stage } => {
@@ -279,7 +292,7 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
                 vec![self.rustc(stage)]
             }
             Source::DocRustc { stage } => {
-                vec![self.doc_std(stage)]
+                vec![self.doc_test(stage)]
             }
             Source::Doc { stage } => {
                 vec![self.doc_book(stage), self.doc_nomicon(stage),
@@ -315,7 +328,7 @@ pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> {
                 vec![self.rustc(stage)]
             }
             Source::DistStd { compiler } => {
-                vec![self.libstd(compiler)]
+                vec![self.libtest(compiler)]
             }
 
             Source::Dist { stage } => {

diff --git a/src/etc/tidy.py b/src/etc/tidy.py
index ce774d31b081fe2a34d727e8d17d291a30376b7a..9264646673b5b4b4bc74b7c7a4abaa7e24349477 100644 (file)
--- a/src/etc/tidy.py
+++ b/src/etc/tidy.py
@@ -31,6 +31,7 @@ stable_whitelist = {
     'src/libcore',
     'src/libstd',
     'src/rustc/std_shim',
+    'src/rustc/test_shim',
     'src/test'
 }
 

diff --git a/src/librustc/Cargo.toml b/src/librustc/Cargo.toml
index 1f000c2af68b880c8310942607cec66b1f07e34d..b446f839d99e6013a7aed93a95a8f299b38e6da3 100644 (file)
--- a/src/librustc/Cargo.toml
+++ b/src/librustc/Cargo.toml
@@ -12,7 +12,6 @@ crate-type = ["dylib"]
 arena = { path = "../libarena" }
 flate = { path = "../libflate" }
 fmt_macros = { path = "../libfmt_macros" }
-getopts = { path = "../libgetopts" }
 graphviz = { path = "../libgraphviz" }
 log = { path = "../liblog" }
 rbml = { path = "../librbml" }

diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml
index c8c517934443d41dd47a9fe0568d53ac46396631..55451eb069eb09c51578611cebbe97d09440fe0b 100644 (file)
--- a/src/librustc_driver/Cargo.toml
+++ b/src/librustc_driver/Cargo.toml
@@ -11,7 +11,6 @@ crate-type = ["dylib"]
 [dependencies]
 arena = { path = "../libarena" }
 flate = { path = "../libflate" }
-getopts = { path = "../libgetopts" }
 graphviz = { path = "../libgraphviz" }
 log = { path = "../liblog" }
 rustc = { path = "../librustc" }

diff --git a/src/librustc_trans/Cargo.toml b/src/librustc_trans/Cargo.toml
index b7faafeba9aa1b9fd66a940f1af24a48a60a3bcb..16e5138b04e18492f5de616a1830bf8226859394 100644 (file)
--- a/src/librustc_trans/Cargo.toml
+++ b/src/librustc_trans/Cargo.toml
@@ -11,7 +11,6 @@ crate-type = ["dylib"]
 [dependencies]
 arena = { path = "../libarena" }
 flate = { path = "../libflate" }
-getopts = { path = "../libgetopts" }
 graphviz = { path = "../libgraphviz" }
 log = { path = "../liblog" }
 rustc = { path = "../librustc" }

diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml
index 6b0ad30f4509c98f3e35aa9c4d5c3d45a003e832..3378b90f87e3890a1a755b3db63b43a182f74523 100644 (file)
--- a/src/librustdoc/Cargo.toml
+++ b/src/librustdoc/Cargo.toml
@@ -11,7 +11,6 @@ crate-type = ["dylib"]
 
 [dependencies]
 arena = { path = "../libarena" }
-getopts = { path = "../libgetopts" }
 rustc = { path = "../librustc" }
 rustc_back = { path = "../librustc_back" }
 rustc_driver = { path = "../librustc_driver" }
@@ -22,7 +21,6 @@ rustc_resolve = { path = "../librustc_resolve" }
 rustc_trans = { path = "../librustc_trans" }
 serialize = { path = "../libserialize" }
 syntax = { path = "../libsyntax" }
-test = { path = "../libtest" }
 log = { path = "../liblog" }
 
 [build-dependencies]

diff --git a/src/libsyntax/Cargo.toml b/src/libsyntax/Cargo.toml
index f5ff7fbf61042baa051d16eb957741f97eddd055..964f2dcb6b6b79119615c42b14b42e9c2e16603e 100644 (file)
--- a/src/libsyntax/Cargo.toml
+++ b/src/libsyntax/Cargo.toml
@@ -10,6 +10,5 @@ crate-type = ["dylib"]
 
 [dependencies]
 serialize = { path = "../libserialize" }
-term = { path = "../libterm" }
 log = { path = "../liblog" }
 rustc_bitflags = { path = "../librustc_bitflags" }

diff --git a/src/rustc/Cargo.lock b/src/rustc/Cargo.lock
index 9ef0b400022989bc92b5761da3a73b48e5fca891..3548e0c8f497e913f8447a4570a8654a60d0db3f 100644 (file)
--- a/src/rustc/Cargo.lock
+++ b/src/rustc/Cargo.lock
@@ -45,10 +45,6 @@ dependencies = [
  "winapi 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "getopts"
-version = "0.0.0"
-
 [[package]]
 name = "graphviz"
 version = "0.0.0"
@@ -72,7 +68,6 @@ dependencies = [
  "arena 0.0.0",
  "flate 0.0.0",
  "fmt_macros 0.0.0",
- "getopts 0.0.0",
  "graphviz 0.0.0",
  "log 0.0.0",
  "rbml 0.0.0",
@@ -136,7 +131,6 @@ version = "0.0.0"
 dependencies = [
  "arena 0.0.0",
  "flate 0.0.0",
- "getopts 0.0.0",
  "graphviz 0.0.0",
  "log 0.0.0",
  "rustc 0.0.0",
@@ -151,6 +145,7 @@ dependencies = [
  "rustc_plugin 0.0.0",
  "rustc_privacy 0.0.0",
  "rustc_resolve 0.0.0",
+ "rustc_save_analysis 0.0.0",
  "rustc_trans 0.0.0",
  "rustc_typeck 0.0.0",
  "serialize 0.0.0",
@@ -273,13 +268,22 @@ dependencies = [
  "syntax 0.0.0",
 ]
 
+[[package]]
+name = "rustc_save_analysis"
+version = "0.0.0"
+dependencies = [
+ "log 0.0.0",
+ "rustc 0.0.0",
+ "rustc_front 0.0.0",
+ "syntax 0.0.0",
+]
+
 [[package]]
 name = "rustc_trans"
 version = "0.0.0"
 dependencies = [
  "arena 0.0.0",
  "flate 0.0.0",
- "getopts 0.0.0",
  "graphviz 0.0.0",
  "log 0.0.0",
  "rustc 0.0.0",
@@ -316,7 +320,6 @@ dependencies = [
  "arena 0.0.0",
  "build_helper 0.1.0",
  "gcc 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
- "getopts 0.0.0",
  "log 0.0.0",
  "rustc 0.0.0",
  "rustc_back 0.0.0",
@@ -328,7 +331,6 @@ dependencies = [
  "rustc_trans 0.0.0",
  "serialize 0.0.0",
  "syntax 0.0.0",
- "test 0.0.0",
 ]
 
 [[package]]
@@ -345,7 +347,6 @@ dependencies = [
  "log 0.0.0",
  "rustc_bitflags 0.0.0",
  "serialize 0.0.0",
- "term 0.0.0",
 ]
 
 [[package]]
@@ -353,21 +354,10 @@ name = "syntax_ext"
 version = "0.0.0"
 dependencies = [
  "fmt_macros 0.0.0",
+ "log 0.0.0",
  "syntax 0.0.0",
 ]
 
-[[package]]
-name = "term"
-version = "0.0.0"
-
-[[package]]
-name = "test"
-version = "0.0.0"
-dependencies = [
- "getopts 0.0.0",
- "term 0.0.0",
-]
-
 [[package]]
 name = "winapi"
 version = "0.2.2"

diff --git a/src/rustc/test_shim/Cargo.lock b/src/rustc/test_shim/Cargo.lock
new file mode 100644 (file)
index 0000000..73df56d
--- /dev/null
+++ b/src/rustc/test_shim/Cargo.lock
@@ -0,0 +1,23 @@
+[root]
+name = "test_shim"
+version = "0.1.0"
+dependencies = [
+ "test 0.0.0",
+]
+
+[[package]]
+name = "getopts"
+version = "0.0.0"
+
+[[package]]
+name = "term"
+version = "0.0.0"
+
+[[package]]
+name = "test"
+version = "0.0.0"
+dependencies = [
+ "getopts 0.0.0",
+ "term 0.0.0",
+]
+

diff --git a/src/rustc/test_shim/Cargo.toml b/src/rustc/test_shim/Cargo.toml
new file mode 100644 (file)
index 0000000..bf57665
--- /dev/null
+++ b/src/rustc/test_shim/Cargo.toml
@@ -0,0 +1,25 @@
+# This is a shim Cargo.toml which serves as a proxy for building libtest.
+#
+# The reason this shim exists is basically the same reason that `std_shim`
+# exists, and more documentation can be found in that `Cargo.toml` as to why.
+
+[package]
+name = "test_shim"
+version = "0.1.0"
+authors = ["The Rust Project Developers"]
+
+[lib]
+name = "test_shim"
+path = "lib.rs"
+
+[profile.release]
+opt-level = 2
+
+# These options are controlled from our rustc wrapper script, so turn them off
+# here and have them controlled elsewhere.
+[profile.dev]
+debug = false
+debug-assertions = false
+
+[dependencies]
+test = { path = "../../libtest" }

diff --git a/src/rustc/test_shim/lib.rs b/src/rustc/test_shim/lib.rs
new file mode 100644 (file)
index 0000000..a626c94
--- /dev/null
+++ b/src/rustc/test_shim/lib.rs
@@ -0,0 +1,11 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// See comments in Cargo.toml for why this exists