]> git.lizzy.rs Git - rust.git/commitdiff
rustbuild: Quickly `dist` cross-host compilers
authorAlex Crichton <alex@alexcrichton.com>
Sat, 31 Dec 2016 03:50:57 +0000 (19:50 -0800)
committerAlex Crichton <alex@alexcrichton.com>
Wed, 4 Jan 2017 19:41:16 +0000 (11:41 -0800)
This commit optimizes the compile time for creating tarballs of cross-host
compilers and as a proof of concept adds two to the standard Travis matrix. Much
of this commit is further refactoring and refining of the `step.rs` definitions
along with the interpretation of `--target` and `--host` flags. This has gotten
confusing enough that I've also added a small test suite to
`src/bootstrap/step.rs` to ensure what we're doing works and doesn't regress.

After this commit when you execute:

    ./x.py dist --host $MY_HOST --target $MY_HOST

the build system will compile two compilers. The first is for the build platform
and the second is for the host platform. This second compiler is then packaged
up and placed into `build/dist` and is ready to go. With a fully cached LLVM and
docker image I was able to create a cross-host compiler in around 20 minutes
locally.

Eventually we plan to add a whole litany of cross-host entries to the Travis
matrix, but for now we're just adding a few before we eat up all the extra
capacity.

cc #38531

.travis.yml
src/bootstrap/Cargo.toml
src/bootstrap/check.rs
src/bootstrap/dist.rs
src/bootstrap/doc.rs
src/bootstrap/step.rs
src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile [new file with mode: 0644]
src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile [new file with mode: 0644]
src/ci/docker/dist-x86_64-unknown-freebsd/build-toolchain.sh [new file with mode: 0644]
src/ci/docker/x86_64-freebsd/Dockerfile [deleted file]
src/ci/docker/x86_64-freebsd/build-toolchain.sh [deleted file]

index 26cabf92bdac81d98ced0ba6cee456f199092458..15b610833b0efd53bfa46a377fc5dda6651bdde1 100644 (file)
@@ -15,9 +15,10 @@ matrix:
     # Linux builders, all docker images
     - env: IMAGE=arm-android
     - env: IMAGE=cross
+    - env: IMAGE=dist-arm-unknown-linux-gnueabi
+    - env: IMAGE=dist-x86_64-unknown-freebsd
     - env: IMAGE=i686-gnu
     - env: IMAGE=i686-gnu-nopt
-    - env: IMAGE=x86_64-freebsd
     - env: IMAGE=x86_64-gnu
     - env: IMAGE=x86_64-gnu-full-bootstrap
     - env: IMAGE=x86_64-gnu-aux
index 35f8fb43f7b562b6321bcd5e3fe1ef425b7aca89..1eda1608c4709c4de84b57b8473ca62cbbaad820 100644 (file)
@@ -6,18 +6,22 @@ version = "0.0.0"
 [lib]
 name = "bootstrap"
 path = "lib.rs"
+doctest = false
 
 [[bin]]
 name = "bootstrap"
 path = "bin/main.rs"
+test = false
 
 [[bin]]
 name = "rustc"
 path = "bin/rustc.rs"
+test = false
 
 [[bin]]
 name = "rustdoc"
 path = "bin/rustdoc.rs"
+test = false
 
 [dependencies]
 build_helper = { path = "../build_helper" }
index 6db1afa54a625493c868ca3c419744c5ccc1124c..f2fddf6e2ef3aee31350322dc826f8ee83f4847e 100644 (file)
@@ -568,3 +568,14 @@ pub fn distcheck(build: &Build) {
                      .arg("check")
                      .current_dir(&dir));
 }
+
+/// Test the build system itself
+pub fn bootstrap(build: &Build) {
+    let mut cmd = Command::new(&build.cargo);
+    cmd.arg("test")
+       .current_dir(build.src.join("src/bootstrap"))
+       .env("CARGO_TARGET_DIR", build.out.join("bootstrap"))
+       .env("RUSTC", &build.rustc);
+    cmd.arg("--").args(&build.flags.cmd.test_args());
+    build.run(&mut cmd);
+}
index 9b0c7a04d6be2e6ec4727013bf33565ca8fee2d3..ad851e448ea7c9e3ceb68cd5be3309e5f84587b6 100644 (file)
@@ -354,14 +354,9 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
 }
 
 /// Creates the `rust-src` installer component and the plain source tarball
-pub fn rust_src(build: &Build, host: &str) {
+pub fn rust_src(build: &Build) {
     println!("Dist src");
 
-    if host != build.config.build {
-        println!("\tskipping, not a build host");
-        return
-    }
-
     let plain_name = format!("rustc-{}-src", package_vers(build));
     let name = format!("rust-src-{}", package_vers(build));
     let image = tmpdir(build).join(format!("{}-image", name));
index bbbf5cba8a1ab23782d87347abb702f48683f07a..42eae6d24f13a0926a4055efc22a4b139f2bbd71 100644 (file)
@@ -57,12 +57,12 @@ pub fn rustbook(build: &Build, target: &str, name: &str) {
 /// `STAMP` alongw ith providing the various header/footer HTML we've cutomized.
 ///
 /// In the end, this is just a glorified wrapper around rustdoc!
-pub fn standalone(build: &Build, stage: u32, target: &str) {
-    println!("Documenting stage{} standalone ({})", stage, target);
+pub fn standalone(build: &Build, target: &str) {
+    println!("Documenting standalone ({})", target);
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
 
-    let compiler = Compiler::new(stage, &build.config.build);
+    let compiler = Compiler::new(0, &build.config.build);
 
     let favicon = build.src.join("src/doc/favicon.inc");
     let footer = build.src.join("src/doc/footer.inc");
index 6a81f759dc73f8e93203d8093dadbf14b68f1f75..bf815a817ed87c4f2483ef2f74fde47f30561a10 100644 (file)
@@ -365,6 +365,8 @@ fn crate_rule<'a, 'b>(build: &'a Build,
 
         suite("check-rpass-full", "src/test/run-pass-fulldeps",
               "run-pass", "run-pass-fulldeps");
+        suite("check-rfail-full", "src/test/run-fail-fulldeps",
+              "run-fail", "run-fail-fulldeps");
         suite("check-cfail-full", "src/test/compile-fail-fulldeps",
               "compile-fail", "compile-fail-fulldeps");
         suite("check-rmake", "src/test/run-make", "run-make", "run-make");
@@ -459,6 +461,7 @@ fn crate_rule<'a, 'b>(build: &'a Build,
          .dep(|s| s.name("tool-tidy").stage(0))
          .default(true)
          .host(true)
+         .only_build(true)
          .run(move |s| check::tidy(build, s.target));
     rules.test("check-error-index", "src/tools/error_index_generator")
          .dep(|s| s.name("libstd"))
@@ -482,6 +485,12 @@ fn crate_rule<'a, 'b>(build: &'a Build,
          .dep(|s| s.name("libtest"))
          .run(move |s| check::android_copy_libs(build, &s.compiler(), s.target));
 
+    rules.test("check-bootstrap", "src/bootstrap")
+         .default(true)
+         .host(true)
+         .only_build(true)
+         .run(move |_| check::bootstrap(build));
+
     // ========================================================================
     // Build tools
     //
@@ -516,9 +525,14 @@ fn crate_rule<'a, 'b>(build: &'a Build,
          .default(build.config.docs)
          .run(move |s| doc::rustbook(build, s.target, "nomicon"));
     rules.doc("doc-standalone", "src/doc")
-         .dep(move |s| s.name("rustc").host(&build.config.build).target(&build.config.build))
+         .dep(move |s| {
+             s.name("rustc")
+              .host(&build.config.build)
+              .target(&build.config.build)
+              .stage(0)
+         })
          .default(build.config.docs)
-         .run(move |s| doc::standalone(build, s.stage, s.target));
+         .run(move |s| doc::standalone(build, s.target));
     rules.doc("doc-error-index", "src/tools/error_index_generator")
          .dep(move |s| s.name("tool-error-index").target(&build.config.build).stage(0))
          .dep(move |s| s.name("librustc-link").stage(0))
@@ -550,6 +564,7 @@ fn crate_rule<'a, 'b>(build: &'a Build,
     rules.dist("dist-rustc", "src/librustc")
          .dep(move |s| s.name("rustc").host(&build.config.build))
          .host(true)
+         .only_host_build(true)
          .default(true)
          .run(move |s| dist::rustc(build, s.stage, s.target));
     rules.dist("dist-std", "src/libstd")
@@ -564,9 +579,11 @@ fn crate_rule<'a, 'b>(build: &'a Build,
              }
          })
          .default(true)
+         .only_host_build(true)
          .run(move |s| dist::std(build, &s.compiler(), s.target));
     rules.dist("dist-mingw", "path/to/nowhere")
          .default(true)
+         .only_host_build(true)
          .run(move |s| {
              if s.target.contains("pc-windows-gnu") {
                  dist::mingw(build, s.target)
@@ -575,14 +592,18 @@ fn crate_rule<'a, 'b>(build: &'a Build,
     rules.dist("dist-src", "src")
          .default(true)
          .host(true)
-         .run(move |s| dist::rust_src(build, s.target));
+         .only_build(true)
+         .only_host_build(true)
+         .run(move |_| dist::rust_src(build));
     rules.dist("dist-docs", "src/doc")
          .default(true)
+         .only_host_build(true)
          .dep(|s| s.name("default:doc"))
          .run(move |s| dist::docs(build, s.stage, s.target));
     rules.dist("dist-analysis", "analysis")
          .dep(|s| s.name("dist-std"))
          .default(true)
+         .only_host_build(true)
          .run(move |s| dist::analysis(build, &s.compiler(), s.target));
     rules.dist("install", "src")
          .dep(|s| s.name("default:dist"))
@@ -671,6 +692,14 @@ struct Rule<'a> {
     /// only intended for compiler hosts and not for targets that are being
     /// generated.
     host: bool,
+
+    /// Whether this rule is only for steps where the host is the build triple,
+    /// not anything in hosts or targets.
+    only_host_build: bool,
+
+    /// Whether this rule is only for the build triple, not anything in hosts or
+    /// targets.
+    only_build: bool,
 }
 
 #[derive(PartialEq)]
@@ -692,6 +721,8 @@ fn new(name: &'a str, path: &'a str, kind: Kind) -> Rule<'a> {
             kind: kind,
             default: false,
             host: false,
+            only_host_build: false,
+            only_build: false,
         }
     }
 }
@@ -727,6 +758,16 @@ fn host(&mut self, host: bool) -> &mut Self {
         self.rule.host = host;
         self
     }
+
+    fn only_build(&mut self, only_build: bool) -> &mut Self {
+        self.rule.only_build = only_build;
+        self
+    }
+
+    fn only_host_build(&mut self, only_host_build: bool) -> &mut Self {
+        self.rule.only_host_build = only_host_build;
+        self
+    }
 }
 
 impl<'a, 'b> Drop for RuleBuilder<'a, 'b> {
@@ -896,19 +937,12 @@ fn plan(&self) -> Vec<Step<'a>> {
                 path.ends_with(rule.path)
             })
         }).flat_map(|rule| {
-            let hosts = if self.build.flags.host.len() > 0 {
+            let hosts = if rule.only_host_build || rule.only_build {
+                &self.build.config.host[..1]
+            } else if self.build.flags.host.len() > 0 {
                 &self.build.flags.host
             } else {
-                if kind == Kind::Dist {
-                    // For 'dist' steps we only distribute artifacts built from
-                    // the build platform, so only consider that in the hosts
-                    // array.
-                    // NOTE: This relies on the fact that the build triple is
-                    // always placed first, as done in `config.rs`.
-                    &self.build.config.host[..1]
-                } else {
-                    &self.build.config.host
-                }
+                &self.build.config.host
             };
             let targets = if self.build.flags.target.len() > 0 {
                 &self.build.flags.target
@@ -928,6 +962,8 @@ fn plan(&self) -> Vec<Step<'a>> {
                     &self.build.flags.host[..]
                 } else if self.build.flags.target.len() > 0 {
                     &[]
+                } else if rule.only_build {
+                    &self.build.config.host[..1]
                 } else {
                     &self.build.config.host[..]
                 }
@@ -955,12 +991,7 @@ fn run(&self, steps: &[Step<'a>]) {
 
         // Using `steps` as the top-level targets, make a topological ordering
         // of what we need to do.
-        let mut order = Vec::new();
-        let mut added = HashSet::new();
-        added.insert(Step::noop());
-        for step in steps.iter().cloned() {
-            self.fill(step, &mut order, &mut added);
-        }
+        let order = self.expand(steps);
 
         // Print out what we're doing for debugging
         self.build.verbose("bootstrap build plan:");
@@ -979,6 +1010,18 @@ fn run(&self, steps: &[Step<'a>]) {
         }
     }
 
+    /// From the top level targets `steps` generate a topological ordering of
+    /// all steps needed to run those steps.
+    fn expand(&self, steps: &[Step<'a>]) -> Vec<Step<'a>> {
+        let mut order = Vec::new();
+        let mut added = HashSet::new();
+        added.insert(Step::noop());
+        for step in steps.iter().cloned() {
+            self.fill(step, &mut order, &mut added);
+        }
+        return order
+    }
+
     /// Performs topological sort of dependencies rooted at the `step`
     /// specified, pushing all results onto the `order` vector provided.
     ///
@@ -1015,3 +1058,367 @@ fn fill(&self,
         order.push(step);
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::env;
+
+    use Build;
+    use config::Config;
+    use flags::Flags;
+
+    macro_rules! a {
+        ($($a:expr),*) => (vec![$($a.to_string()),*])
+    }
+
+    fn build(args: &[&str],
+             extra_host: &[&str],
+             extra_target: &[&str]) -> Build {
+        let mut args = args.iter().map(|s| s.to_string()).collect::<Vec<_>>();
+        args.push("--build".to_string());
+        args.push("A".to_string());
+        let flags = Flags::parse(&args);
+
+        let mut config = Config::default();
+        config.docs = true;
+        config.build = "A".to_string();
+        config.host = vec![config.build.clone()];
+        config.host.extend(extra_host.iter().map(|s| s.to_string()));
+        config.target = config.host.clone();
+        config.target.extend(extra_target.iter().map(|s| s.to_string()));
+
+        let mut build = Build::new(flags, config);
+        let cwd = env::current_dir().unwrap();
+        build.crates.insert("std_shim".to_string(), ::Crate {
+            name: "std_shim".to_string(),
+            deps: Vec::new(),
+            path: cwd.join("src/std_shim"),
+            doc_step: "doc-std_shim".to_string(),
+            build_step: "build-crate-std_shim".to_string(),
+            test_step: "test-std_shim".to_string(),
+            bench_step: "bench-std_shim".to_string(),
+        });
+        build.crates.insert("test_shim".to_string(), ::Crate {
+            name: "test_shim".to_string(),
+            deps: Vec::new(),
+            path: cwd.join("src/test_shim"),
+            doc_step: "doc-test_shim".to_string(),
+            build_step: "build-crate-test_shim".to_string(),
+            test_step: "test-test_shim".to_string(),
+            bench_step: "bench-test_shim".to_string(),
+        });
+        build.crates.insert("rustc-main".to_string(), ::Crate {
+            name: "rustc-main".to_string(),
+            deps: Vec::new(),
+            path: cwd.join("src/rustc-main"),
+            doc_step: "doc-rustc-main".to_string(),
+            build_step: "build-crate-rustc-main".to_string(),
+            test_step: "test-rustc-main".to_string(),
+            bench_step: "bench-rustc-main".to_string(),
+        });
+        return build
+    }
+
+    #[test]
+    fn dist_baseline() {
+        let build = build(&["dist"], &[], &[]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+        assert!(plan.iter().all(|s| s.host == "A" ));
+        assert!(plan.iter().all(|s| s.target == "A" ));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        assert!(plan.contains(&step.name("dist-docs")));
+        assert!(plan.contains(&step.name("dist-mingw")));
+        assert!(plan.contains(&step.name("dist-rustc")));
+        assert!(plan.contains(&step.name("dist-std")));
+        assert!(plan.contains(&step.name("dist-src")));
+    }
+
+    #[test]
+    fn dist_with_targets() {
+        let build = build(&["dist"], &[], &["B"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+        assert!(plan.iter().all(|s| s.host == "A" ));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        assert!(plan.contains(&step.name("dist-docs")));
+        assert!(plan.contains(&step.name("dist-mingw")));
+        assert!(plan.contains(&step.name("dist-rustc")));
+        assert!(plan.contains(&step.name("dist-std")));
+        assert!(plan.contains(&step.name("dist-src")));
+
+        assert!(plan.contains(&step.target("B").name("dist-docs")));
+        assert!(plan.contains(&step.target("B").name("dist-mingw")));
+        assert!(!plan.contains(&step.target("B").name("dist-rustc")));
+        assert!(plan.contains(&step.target("B").name("dist-std")));
+        assert!(!plan.contains(&step.target("B").name("dist-src")));
+    }
+
+    #[test]
+    fn dist_with_hosts() {
+        let build = build(&["dist"], &["B"], &[]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        assert!(!plan.iter().any(|s| s.host == "B"));
+
+        assert!(plan.contains(&step.name("dist-docs")));
+        assert!(plan.contains(&step.name("dist-mingw")));
+        assert!(plan.contains(&step.name("dist-rustc")));
+        assert!(plan.contains(&step.name("dist-std")));
+        assert!(plan.contains(&step.name("dist-src")));
+
+        assert!(plan.contains(&step.target("B").name("dist-docs")));
+        assert!(plan.contains(&step.target("B").name("dist-mingw")));
+        assert!(plan.contains(&step.target("B").name("dist-rustc")));
+        assert!(plan.contains(&step.target("B").name("dist-std")));
+        assert!(!plan.contains(&step.target("B").name("dist-src")));
+    }
+
+    #[test]
+    fn dist_with_targets_and_hosts() {
+        let build = build(&["dist"], &["B"], &["C"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        assert!(!plan.iter().any(|s| s.host == "B"));
+        assert!(!plan.iter().any(|s| s.host == "C"));
+
+        assert!(plan.contains(&step.name("dist-docs")));
+        assert!(plan.contains(&step.name("dist-mingw")));
+        assert!(plan.contains(&step.name("dist-rustc")));
+        assert!(plan.contains(&step.name("dist-std")));
+        assert!(plan.contains(&step.name("dist-src")));
+
+        assert!(plan.contains(&step.target("B").name("dist-docs")));
+        assert!(plan.contains(&step.target("B").name("dist-mingw")));
+        assert!(plan.contains(&step.target("B").name("dist-rustc")));
+        assert!(plan.contains(&step.target("B").name("dist-std")));
+        assert!(!plan.contains(&step.target("B").name("dist-src")));
+
+        assert!(plan.contains(&step.target("C").name("dist-docs")));
+        assert!(plan.contains(&step.target("C").name("dist-mingw")));
+        assert!(!plan.contains(&step.target("C").name("dist-rustc")));
+        assert!(plan.contains(&step.target("C").name("dist-std")));
+        assert!(!plan.contains(&step.target("C").name("dist-src")));
+    }
+
+    #[test]
+    fn dist_target_with_target_flag() {
+        let build = build(&["dist", "--target=C"], &["B"], &["C"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        assert!(!plan.iter().any(|s| s.target == "A"));
+        assert!(!plan.iter().any(|s| s.target == "B"));
+        assert!(!plan.iter().any(|s| s.host == "B"));
+        assert!(!plan.iter().any(|s| s.host == "C"));
+
+        assert!(plan.contains(&step.target("C").name("dist-docs")));
+        assert!(plan.contains(&step.target("C").name("dist-mingw")));
+        assert!(!plan.contains(&step.target("C").name("dist-rustc")));
+        assert!(plan.contains(&step.target("C").name("dist-std")));
+        assert!(!plan.contains(&step.target("C").name("dist-src")));
+    }
+
+    #[test]
+    fn dist_host_with_target_flag() {
+        let build = build(&["dist", "--host=B", "--target=B"], &["B"], &["C"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        assert!(!plan.iter().any(|s| s.target == "A"));
+        assert!(!plan.iter().any(|s| s.target == "C"));
+        assert!(!plan.iter().any(|s| s.host == "B"));
+        assert!(!plan.iter().any(|s| s.host == "C"));
+
+        assert!(plan.contains(&step.target("B").name("dist-docs")));
+        assert!(plan.contains(&step.target("B").name("dist-mingw")));
+        assert!(plan.contains(&step.target("B").name("dist-rustc")));
+        assert!(plan.contains(&step.target("B").name("dist-std")));
+        assert!(plan.contains(&step.target("B").name("dist-src")));
+
+        let all = rules.expand(&plan);
+        println!("all rules: {:#?}", all);
+        assert!(!all.contains(&step.name("rustc")));
+        assert!(!all.contains(&step.name("build-crate-std_shim").stage(1)));
+    }
+
+    #[test]
+    fn build_default() {
+        let build = build(&["build"], &["B"], &["C"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+
+        let step = super::Step {
+            name: "",
+            stage: 2,
+            host: &build.config.build,
+            target: &build.config.build,
+        };
+
+        // rustc built for all for of (A, B) x (A, B)
+        assert!(plan.contains(&step.name("librustc")));
+        assert!(plan.contains(&step.target("B").name("librustc")));
+        assert!(plan.contains(&step.host("B").target("A").name("librustc")));
+        assert!(plan.contains(&step.host("B").target("B").name("librustc")));
+
+        // rustc never built for C
+        assert!(!plan.iter().any(|s| {
+            s.name.contains("rustc") && (s.host == "C" || s.target == "C")
+        }));
+
+        // test built for everything
+        assert!(plan.contains(&step.name("libtest")));
+        assert!(plan.contains(&step.target("B").name("libtest")));
+        assert!(plan.contains(&step.host("B").target("A").name("libtest")));
+        assert!(plan.contains(&step.host("B").target("B").name("libtest")));
+        assert!(plan.contains(&step.host("A").target("C").name("libtest")));
+        assert!(plan.contains(&step.host("B").target("C").name("libtest")));
+
+        let all = rules.expand(&plan);
+        println!("all rules: {:#?}", all);
+        assert!(all.contains(&step.name("rustc")));
+        assert!(all.contains(&step.name("libstd")));
+    }
+
+    #[test]
+    fn build_filtered() {
+        let build = build(&["build", "--target=C"], &["B"], &["C"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+
+        assert!(!plan.iter().any(|s| s.name.contains("rustc")));
+        assert!(plan.iter().all(|s| {
+            !s.name.contains("test_shim") || s.target == "C"
+        }));
+    }
+
+    #[test]
+    fn test_default() {
+        let build = build(&["test"], &[], &[]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+        assert!(plan.iter().all(|s| s.host == "A"));
+        assert!(plan.iter().all(|s| s.target == "A"));
+
+        assert!(plan.iter().any(|s| s.name.contains("-ui")));
+        assert!(plan.iter().any(|s| s.name.contains("cfail")));
+        assert!(plan.iter().any(|s| s.name.contains("cfail")));
+        assert!(plan.iter().any(|s| s.name.contains("cfail-full")));
+        assert!(plan.iter().any(|s| s.name.contains("codegen-units")));
+        assert!(plan.iter().any(|s| s.name.contains("debuginfo")));
+        assert!(plan.iter().any(|s| s.name.contains("docs")));
+        assert!(plan.iter().any(|s| s.name.contains("error-index")));
+        assert!(plan.iter().any(|s| s.name.contains("incremental")));
+        assert!(plan.iter().any(|s| s.name.contains("linkchecker")));
+        assert!(plan.iter().any(|s| s.name.contains("mir-opt")));
+        assert!(plan.iter().any(|s| s.name.contains("pfail")));
+        assert!(plan.iter().any(|s| s.name.contains("rfail")));
+        assert!(plan.iter().any(|s| s.name.contains("rfail-full")));
+        assert!(plan.iter().any(|s| s.name.contains("rmake")));
+        assert!(plan.iter().any(|s| s.name.contains("rpass")));
+        assert!(plan.iter().any(|s| s.name.contains("rpass-full")));
+        assert!(plan.iter().any(|s| s.name.contains("rustc-all")));
+        assert!(plan.iter().any(|s| s.name.contains("rustdoc")));
+        assert!(plan.iter().any(|s| s.name.contains("std-all")));
+        assert!(plan.iter().any(|s| s.name.contains("test-all")));
+        assert!(plan.iter().any(|s| s.name.contains("tidy")));
+        assert!(plan.iter().any(|s| s.name.contains("valgrind")));
+    }
+
+    #[test]
+    fn test_with_a_target() {
+        let build = build(&["test", "--target=C"], &[], &["C"]);
+        let rules = super::build_rules(&build);
+        let plan = rules.plan();
+        println!("rules: {:#?}", plan);
+        assert!(plan.iter().all(|s| s.stage == 2));
+        assert!(plan.iter().all(|s| s.host == "A"));
+        assert!(plan.iter().all(|s| s.target == "C"));
+
+        assert!(plan.iter().any(|s| s.name.contains("-ui")));
+        assert!(plan.iter().any(|s| s.name.contains("cfail")));
+        assert!(plan.iter().any(|s| s.name.contains("cfail")));
+        assert!(!plan.iter().any(|s| s.name.contains("cfail-full")));
+        assert!(plan.iter().any(|s| s.name.contains("codegen-units")));
+        assert!(plan.iter().any(|s| s.name.contains("debuginfo")));
+        assert!(!plan.iter().any(|s| s.name.contains("docs")));
+        assert!(!plan.iter().any(|s| s.name.contains("error-index")));
+        assert!(plan.iter().any(|s| s.name.contains("incremental")));
+        assert!(!plan.iter().any(|s| s.name.contains("linkchecker")));
+        assert!(plan.iter().any(|s| s.name.contains("mir-opt")));
+        assert!(plan.iter().any(|s| s.name.contains("pfail")));
+        assert!(plan.iter().any(|s| s.name.contains("rfail")));
+        assert!(!plan.iter().any(|s| s.name.contains("rfail-full")));
+        assert!(!plan.iter().any(|s| s.name.contains("rmake")));
+        assert!(plan.iter().any(|s| s.name.contains("rpass")));
+        assert!(!plan.iter().any(|s| s.name.contains("rpass-full")));
+        assert!(!plan.iter().any(|s| s.name.contains("rustc-all")));
+        assert!(!plan.iter().any(|s| s.name.contains("rustdoc")));
+        assert!(plan.iter().any(|s| s.name.contains("std-all")));
+        assert!(plan.iter().any(|s| s.name.contains("test-all")));
+        assert!(!plan.iter().any(|s| s.name.contains("tidy")));
+        assert!(plan.iter().any(|s| s.name.contains("valgrind")));
+    }
+}
diff --git a/src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile b/src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile
new file mode 100644 (file)
index 0000000..9b0f1b7
--- /dev/null
@@ -0,0 +1,30 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils \
+  g++-arm-linux-gnueabi
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
+RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    rm dumb-init_*.deb
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi
+ENV XPY_RUN \
+      dist \
+      --host arm-unknown-linux-gnueabi \
+      --target arm-unknown-linux-gnueabi
diff --git a/src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile b/src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile
new file mode 100644 (file)
index 0000000..f1a6ccf
--- /dev/null
@@ -0,0 +1,38 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  bzip2 \
+  xz-utils \
+  wget
+
+COPY build-toolchain.sh /tmp/
+RUN sh /tmp/build-toolchain.sh
+
+RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    rm dumb-init_*.deb
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
+ENV \
+    AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
+    CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc \
+    CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++
+
+ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd
+ENV XPY_RUN \
+      dist \
+      --host x86_64-unknown-freebsd \
+      --target x86_64-unknown-freebsd
diff --git a/src/ci/docker/dist-x86_64-unknown-freebsd/build-toolchain.sh b/src/ci/docker/dist-x86_64-unknown-freebsd/build-toolchain.sh
new file mode 100644 (file)
index 0000000..0fd6bea
--- /dev/null
@@ -0,0 +1,96 @@
+#!/bin/bash
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+ARCH=x86_64
+BINUTILS=2.25.1
+GCC=5.3.0
+
+mkdir binutils
+cd binutils
+
+# First up, build binutils
+curl https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS.tar.bz2 | tar xjf -
+mkdir binutils-build
+cd binutils-build
+../binutils-$BINUTILS/configure \
+  --target=$ARCH-unknown-freebsd10
+make -j10
+make install
+cd ../..
+rm -rf binutils
+
+# Next, download the FreeBSD libc and relevant header files
+
+mkdir freebsd
+case "$ARCH" in
+    x86_64)
+        URL=ftp://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/base.txz
+        ;;
+    i686)
+        URL=ftp://ftp.freebsd.org/pub/FreeBSD/releases/i386/10.2-RELEASE/base.txz
+        ;;
+esac
+curl $URL | tar xJf - -C freebsd ./usr/include ./usr/lib ./lib
+
+dst=/usr/local/$ARCH-unknown-freebsd10
+
+cp -r freebsd/usr/include $dst/
+cp freebsd/usr/lib/crt1.o $dst/lib
+cp freebsd/usr/lib/Scrt1.o $dst/lib
+cp freebsd/usr/lib/crti.o $dst/lib
+cp freebsd/usr/lib/crtn.o $dst/lib
+cp freebsd/usr/lib/libc.a $dst/lib
+cp freebsd/usr/lib/libutil.a $dst/lib
+cp freebsd/usr/lib/libutil_p.a $dst/lib
+cp freebsd/usr/lib/libm.a $dst/lib
+cp freebsd/usr/lib/librt.so.1 $dst/lib
+cp freebsd/usr/lib/libexecinfo.so.1 $dst/lib
+cp freebsd/lib/libc.so.7 $dst/lib
+cp freebsd/lib/libm.so.5 $dst/lib
+cp freebsd/lib/libutil.so.9 $dst/lib
+cp freebsd/lib/libthr.so.3 $dst/lib/libpthread.so
+
+ln -s libc.so.7 $dst/lib/libc.so
+ln -s libm.so.5 $dst/lib/libm.so
+ln -s librt.so.1 $dst/lib/librt.so
+ln -s libutil.so.9 $dst/lib/libutil.so
+ln -s libexecinfo.so.1 $dst/lib/libexecinfo.so
+rm -rf freebsd
+
+# Finally, download and build gcc to target FreeBSD
+mkdir gcc
+cd gcc
+curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.bz2 | tar xjf -
+cd gcc-$GCC
+./contrib/download_prerequisites
+
+mkdir ../gcc-build
+cd ../gcc-build
+../gcc-$GCC/configure                            \
+  --enable-languages=c,c++                       \
+  --target=$ARCH-unknown-freebsd10               \
+  --disable-multilib                             \
+  --disable-nls                                  \
+  --disable-libgomp                              \
+  --disable-libquadmath                          \
+  --disable-libssp                               \
+  --disable-libvtv                               \
+  --disable-libcilkrts                           \
+  --disable-libada                               \
+  --disable-libsanitizer                         \
+  --disable-libquadmath-support                  \
+  --disable-lto
+make -j10
+make install
+cd ../..
+rm -rf gcc
diff --git a/src/ci/docker/x86_64-freebsd/Dockerfile b/src/ci/docker/x86_64-freebsd/Dockerfile
deleted file mode 100644 (file)
index 86efa74..0000000
+++ /dev/null
@@ -1,34 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  g++ \
-  make \
-  file \
-  curl \
-  ca-certificates \
-  python2.7 \
-  git \
-  cmake \
-  sudo \
-  bzip2 \
-  xz-utils \
-  wget
-
-COPY build-toolchain.sh /tmp/
-RUN sh /tmp/build-toolchain.sh
-
-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
-ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
-RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
-      tar xJf - -C /usr/local/bin --strip-components=1
-
-ENV \
-    AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
-    CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc
-
-ENV RUST_CONFIGURE_ARGS --target=x86_64-unknown-freebsd
-ENV RUST_CHECK_TARGET ""
diff --git a/src/ci/docker/x86_64-freebsd/build-toolchain.sh b/src/ci/docker/x86_64-freebsd/build-toolchain.sh
deleted file mode 100644 (file)
index d4bc886..0000000
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/bin/bash
-# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution and at
-# http://rust-lang.org/COPYRIGHT.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-
-set -ex
-
-ARCH=x86_64
-BINUTILS=2.25.1
-GCC=5.3.0
-
-mkdir binutils
-cd binutils
-
-# First up, build binutils
-curl https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS.tar.bz2 | tar xjf -
-mkdir binutils-build
-cd binutils-build
-../binutils-$BINUTILS/configure \
-  --target=$ARCH-unknown-freebsd10
-make -j10
-make install
-cd ../..
-rm -rf binutils
-
-# Next, download the FreeBSD libc and relevant header files
-
-mkdir freebsd
-case "$ARCH" in
-    x86_64)
-        URL=ftp://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/base.txz
-        ;;
-    i686)
-        URL=ftp://ftp.freebsd.org/pub/FreeBSD/releases/i386/10.2-RELEASE/base.txz
-        ;;
-esac
-curl $URL | tar xJf - -C freebsd ./usr/include ./usr/lib ./lib
-
-dst=/usr/local/$ARCH-unknown-freebsd10
-
-cp -r freebsd/usr/include $dst/
-cp freebsd/usr/lib/crt1.o $dst/lib
-cp freebsd/usr/lib/Scrt1.o $dst/lib
-cp freebsd/usr/lib/crti.o $dst/lib
-cp freebsd/usr/lib/crtn.o $dst/lib
-cp freebsd/usr/lib/libc.a $dst/lib
-cp freebsd/usr/lib/libutil.a $dst/lib
-cp freebsd/usr/lib/libutil_p.a $dst/lib
-cp freebsd/usr/lib/libm.a $dst/lib
-cp freebsd/usr/lib/librt.so.1 $dst/lib
-cp freebsd/usr/lib/libexecinfo.so.1 $dst/lib
-cp freebsd/lib/libc.so.7 $dst/lib
-cp freebsd/lib/libm.so.5 $dst/lib
-cp freebsd/lib/libutil.so.9 $dst/lib
-cp freebsd/lib/libthr.so.3 $dst/lib/libpthread.so
-
-ln -s libc.so.7 $dst/lib/libc.so
-ln -s libm.so.5 $dst/lib/libm.so
-ln -s librt.so.1 $dst/lib/librt.so
-ln -s libutil.so.9 $dst/lib/libutil.so
-ln -s libexecinfo.so.1 $dst/lib/libexecinfo.so
-rm -rf freebsd
-
-# Finally, download and build gcc to target FreeBSD
-mkdir gcc
-cd gcc
-curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.bz2 | tar xjf -
-cd gcc-$GCC
-./contrib/download_prerequisites
-
-mkdir ../gcc-build
-cd ../gcc-build
-../gcc-$GCC/configure                            \
-  --enable-languages=c                           \
-  --target=$ARCH-unknown-freebsd10               \
-  --disable-multilib                             \
-  --disable-nls                                  \
-  --disable-libgomp                              \
-  --disable-libquadmath                          \
-  --disable-libssp                               \
-  --disable-libvtv                               \
-  --disable-libcilkrts                           \
-  --disable-libada                               \
-  --disable-libsanitizer                         \
-  --disable-libquadmath-support                  \
-  --disable-lto
-make -j10
-make install
-cd ../..
-rm -rf gcc