src/bootstrap/test.rs

   1 //! Implementation of the test-related targets of the build system.
   2 //!
   3 //! This file implements the various regression test suites that we execute on
   4 //! our CI.
   5
   6 use std::env;
   7 use std::ffi::OsString;
   8 use std::fmt;
   9 use std::fs;
  10 use std::iter;
  11 use std::path::{Path, PathBuf};
  12 use std::process::Command;
  13
  14 use build_helper::{self, output, t};
  15
  16 use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step};
  17 use crate::cache::{Interned, INTERNER};
  18 use crate::compile;
  19 use crate::dist;
  20 use crate::flags::Subcommand;
  21 use crate::native;
  22 use crate::tool::{self, Tool, SourceType};
  23 use crate::toolstate::ToolState;
  24 use crate::util::{self, dylib_path, dylib_path_var};
  25 use crate::Crate as CargoCrate;
  26 use crate::{DocTests, Mode, GitRepo};
  27
  28 const ADB_TEST_DIR: &str = "/data/tmp/work";
  29
  30 /// The two modes of the test runner; tests or benchmarks.
  31 #[derive(Debug, PartialEq, Eq, Hash, Copy, Clone, PartialOrd, Ord)]
  32 pub enum TestKind {
  33     /// Run `cargo test`.
  34     Test,
  35     /// Run `cargo bench`.
  36     Bench,
  37 }
  38
  39 impl From<Kind> for TestKind {
  40     fn from(kind: Kind) -> Self {
  41         match kind {
  42             Kind::Test => TestKind::Test,
  43             Kind::Bench => TestKind::Bench,
  44             _ => panic!("unexpected kind in crate: {:?}", kind),
  45         }
  46     }
  47 }
  48
  49 impl TestKind {
  50     // Return the cargo subcommand for this test kind
  51     fn subcommand(self) -> &'static str {
  52         match self {
  53             TestKind::Test => "test",
  54             TestKind::Bench => "bench",
  55         }
  56     }
  57 }
  58
  59 impl fmt::Display for TestKind {
  60     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  61         f.write_str(match *self {
  62             TestKind::Test => "Testing",
  63             TestKind::Bench => "Benchmarking",
  64         })
  65     }
  66 }
  67
  68 fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
  69     if !builder.fail_fast {
  70         if !builder.try_run(cmd) {
  71             let mut failures = builder.delayed_failures.borrow_mut();
  72             failures.push(format!("{:?}", cmd));
  73             return false;
  74         }
  75     } else {
  76         builder.run(cmd);
  77     }
  78     true
  79 }
  80
  81 fn try_run_quiet(builder: &Builder<'_>, cmd: &mut Command) -> bool {
  82     if !builder.fail_fast {
  83         if !builder.try_run_quiet(cmd) {
  84             let mut failures = builder.delayed_failures.borrow_mut();
  85             failures.push(format!("{:?}", cmd));
  86             return false;
  87         }
  88     } else {
  89         builder.run_quiet(cmd);
  90     }
  91     true
  92 }
  93
  94 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
  95 pub struct Linkcheck {
  96     host: Interned<String>,
  97 }
  98
  99 impl Step for Linkcheck {
 100     type Output = ();
 101     const ONLY_HOSTS: bool = true;
 102     const DEFAULT: bool = true;
 103
 104     /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
 105     ///
 106     /// This tool in `src/tools` will verify the validity of all our links in the
 107     /// documentation to ensure we don't have a bunch of dead ones.
 108     fn run(self, builder: &Builder<'_>) {
 109         let host = self.host;
 110
 111         builder.info(&format!("Linkcheck ({})", host));
 112
 113         builder.default_doc(None);
 114
 115         let _time = util::timeit(&builder);
 116         try_run(
 117             builder,
 118             builder
 119                 .tool_cmd(Tool::Linkchecker)
 120                 .arg(builder.out.join(host).join("doc")),
 121         );
 122     }
 123
 124     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 125         let builder = run.builder;
 126         run.path("src/tools/linkchecker")
 127             .default_condition(builder.config.docs)
 128     }
 129
 130     fn make_run(run: RunConfig<'_>) {
 131         run.builder.ensure(Linkcheck { host: run.target });
 132     }
 133 }
 134
 135 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 136 pub struct Cargotest {
 137     stage: u32,
 138     host: Interned<String>,
 139 }
 140
 141 impl Step for Cargotest {
 142     type Output = ();
 143     const ONLY_HOSTS: bool = true;
 144
 145     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 146         run.path("src/tools/cargotest")
 147     }
 148
 149     fn make_run(run: RunConfig<'_>) {
 150         run.builder.ensure(Cargotest {
 151             stage: run.builder.top_stage,
 152             host: run.target,
 153         });
 154     }
 155
 156     /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler.
 157     ///
 158     /// This tool in `src/tools` will check out a few Rust projects and run `cargo
 159     /// test` to ensure that we don't regress the test suites there.
 160     fn run(self, builder: &Builder<'_>) {
 161         let compiler = builder.compiler(self.stage, self.host);
 162         builder.ensure(compile::Rustc {
 163             compiler,
 164             target: compiler.host,
 165         });
 166
 167         // Note that this is a short, cryptic, and not scoped directory name. This
 168         // is currently to minimize the length of path on Windows where we otherwise
 169         // quickly run into path name limit constraints.
 170         let out_dir = builder.out.join("ct");
 171         t!(fs::create_dir_all(&out_dir));
 172
 173         let _time = util::timeit(&builder);
 174         let mut cmd = builder.tool_cmd(Tool::CargoTest);
 175         try_run(
 176             builder,
 177             cmd.arg(&builder.initial_cargo)
 178                 .arg(&out_dir)
 179                 .env("RUSTC", builder.rustc(compiler))
 180                 .env("RUSTDOC", builder.rustdoc(compiler)),
 181         );
 182     }
 183 }
 184
 185 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 186 pub struct Cargo {
 187     stage: u32,
 188     host: Interned<String>,
 189 }
 190
 191 impl Step for Cargo {
 192     type Output = ();
 193     const ONLY_HOSTS: bool = true;
 194
 195     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 196         run.path("src/tools/cargo")
 197     }
 198
 199     fn make_run(run: RunConfig<'_>) {
 200         run.builder.ensure(Cargo {
 201             stage: run.builder.top_stage,
 202             host: run.target,
 203         });
 204     }
 205
 206     /// Runs `cargo test` for `cargo` packaged with Rust.
 207     fn run(self, builder: &Builder<'_>) {
 208         let compiler = builder.compiler(self.stage, self.host);
 209
 210         builder.ensure(tool::Cargo {
 211             compiler,
 212             target: self.host,
 213         });
 214         let mut cargo = tool::prepare_tool_cargo(builder,
 215                                                  compiler,
 216                                                  Mode::ToolRustc,
 217                                                  self.host,
 218                                                  "test",
 219                                                  "src/tools/cargo",
 220                                                  SourceType::Submodule,
 221                                                  &[]);
 222
 223         if !builder.fail_fast {
 224             cargo.arg("--no-fail-fast");
 225         }
 226
 227         // Don't run cross-compile tests, we may not have cross-compiled libstd libs
 228         // available.
 229         cargo.env("CFG_DISABLE_CROSS_TESTS", "1");
 230         // Disable a test that has issues with mingw.
 231         cargo.env("CARGO_TEST_DISABLE_GIT_CLI", "1");
 232         // Forcibly disable tests using nightly features since any changes to
 233         // those features won't be able to land.
 234         cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1");
 235
 236         try_run(
 237             builder,
 238             cargo.env("PATH", &path_for_cargo(builder, compiler)),
 239         );
 240     }
 241 }
 242
 243 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 244 pub struct Rls {
 245     stage: u32,
 246     host: Interned<String>,
 247 }
 248
 249 impl Step for Rls {
 250     type Output = ();
 251     const ONLY_HOSTS: bool = true;
 252
 253     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 254         run.path("src/tools/rls")
 255     }
 256
 257     fn make_run(run: RunConfig<'_>) {
 258         run.builder.ensure(Rls {
 259             stage: run.builder.top_stage,
 260             host: run.target,
 261         });
 262     }
 263
 264     /// Runs `cargo test` for the rls.
 265     fn run(self, builder: &Builder<'_>) {
 266         let stage = self.stage;
 267         let host = self.host;
 268         let compiler = builder.compiler(stage, host);
 269
 270         let build_result = builder.ensure(tool::Rls {
 271             compiler,
 272             target: self.host,
 273             extra_features: Vec::new(),
 274         });
 275         if build_result.is_none() {
 276             eprintln!("failed to test rls: could not build");
 277             return;
 278         }
 279
 280         let mut cargo = tool::prepare_tool_cargo(builder,
 281                                                  compiler,
 282                                                  Mode::ToolRustc,
 283                                                  host,
 284                                                  "test",
 285                                                  "src/tools/rls",
 286                                                  SourceType::Submodule,
 287                                                  &[]);
 288
 289         builder.add_rustc_lib_path(compiler, &mut cargo);
 290         cargo.arg("--")
 291             .args(builder.config.cmd.test_args());
 292
 293         if try_run(builder, &mut cargo) {
 294             builder.save_toolstate("rls", ToolState::TestPass);
 295         }
 296     }
 297 }
 298
 299 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 300 pub struct Rustfmt {
 301     stage: u32,
 302     host: Interned<String>,
 303 }
 304
 305 impl Step for Rustfmt {
 306     type Output = ();
 307     const ONLY_HOSTS: bool = true;
 308
 309     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 310         run.path("src/tools/rustfmt")
 311     }
 312
 313     fn make_run(run: RunConfig<'_>) {
 314         run.builder.ensure(Rustfmt {
 315             stage: run.builder.top_stage,
 316             host: run.target,
 317         });
 318     }
 319
 320     /// Runs `cargo test` for rustfmt.
 321     fn run(self, builder: &Builder<'_>) {
 322         let stage = self.stage;
 323         let host = self.host;
 324         let compiler = builder.compiler(stage, host);
 325
 326         let build_result = builder.ensure(tool::Rustfmt {
 327             compiler,
 328             target: self.host,
 329             extra_features: Vec::new(),
 330         });
 331         if build_result.is_none() {
 332             eprintln!("failed to test rustfmt: could not build");
 333             return;
 334         }
 335
 336         let mut cargo = tool::prepare_tool_cargo(builder,
 337                                                  compiler,
 338                                                  Mode::ToolRustc,
 339                                                  host,
 340                                                  "test",
 341                                                  "src/tools/rustfmt",
 342                                                  SourceType::Submodule,
 343                                                  &[]);
 344
 345         let dir = testdir(builder, compiler.host);
 346         t!(fs::create_dir_all(&dir));
 347         cargo.env("RUSTFMT_TEST_DIR", dir);
 348
 349         builder.add_rustc_lib_path(compiler, &mut cargo);
 350
 351         if try_run(builder, &mut cargo) {
 352             builder.save_toolstate("rustfmt", ToolState::TestPass);
 353         }
 354     }
 355 }
 356
 357 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 358 pub struct Miri {
 359     stage: u32,
 360     host: Interned<String>,
 361 }
 362
 363 impl Step for Miri {
 364     type Output = ();
 365     const ONLY_HOSTS: bool = true;
 366     const DEFAULT: bool = true;
 367
 368     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 369         let test_miri = run.builder.config.test_miri;
 370         run.path("src/tools/miri").default_condition(test_miri)
 371     }
 372
 373     fn make_run(run: RunConfig<'_>) {
 374         run.builder.ensure(Miri {
 375             stage: run.builder.top_stage,
 376             host: run.target,
 377         });
 378     }
 379
 380     /// Runs `cargo test` for miri.
 381     fn run(self, builder: &Builder<'_>) {
 382         let stage = self.stage;
 383         let host = self.host;
 384         let compiler = builder.compiler(stage, host);
 385
 386         let miri = builder.ensure(tool::Miri {
 387             compiler,
 388             target: self.host,
 389             extra_features: Vec::new(),
 390         });
 391         if let Some(miri) = miri {
 392             let mut cargo = tool::prepare_tool_cargo(builder,
 393                                                  compiler,
 394                                                  Mode::ToolRustc,
 395                                                  host,
 396                                                  "test",
 397                                                  "src/tools/miri",
 398                                                  SourceType::Submodule,
 399                                                  &[]);
 400
 401             // miri tests need to know about the stage sysroot
 402             cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
 403             cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
 404             cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
 405             cargo.env("MIRI_PATH", miri);
 406
 407             builder.add_rustc_lib_path(compiler, &mut cargo);
 408
 409             if try_run(builder, &mut cargo) {
 410                 builder.save_toolstate("miri", ToolState::TestPass);
 411             }
 412         } else {
 413             eprintln!("failed to test miri: could not build");
 414         }
 415     }
 416 }
 417
 418 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 419 pub struct CompiletestTest {
 420     host: Interned<String>,
 421 }
 422
 423 impl Step for CompiletestTest {
 424     type Output = ();
 425
 426     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 427         run.path("src/tools/compiletest")
 428     }
 429
 430     fn make_run(run: RunConfig<'_>) {
 431         run.builder.ensure(CompiletestTest {
 432             host: run.target,
 433         });
 434     }
 435
 436     /// Runs `cargo test` for compiletest.
 437     fn run(self, builder: &Builder<'_>) {
 438         let host = self.host;
 439         let compiler = builder.compiler(0, host);
 440
 441         let mut cargo = tool::prepare_tool_cargo(builder,
 442                                                  compiler,
 443                                                  Mode::ToolBootstrap,
 444                                                  host,
 445                                                  "test",
 446                                                  "src/tools/compiletest",
 447                                                  SourceType::InTree,
 448                                                  &[]);
 449
 450         try_run(builder, &mut cargo);
 451     }
 452 }
 453
 454 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 455 pub struct Clippy {
 456     stage: u32,
 457     host: Interned<String>,
 458 }
 459
 460 impl Step for Clippy {
 461     type Output = ();
 462     const ONLY_HOSTS: bool = true;
 463     const DEFAULT: bool = false;
 464
 465     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 466         run.path("src/tools/clippy")
 467     }
 468
 469     fn make_run(run: RunConfig<'_>) {
 470         run.builder.ensure(Clippy {
 471             stage: run.builder.top_stage,
 472             host: run.target,
 473         });
 474     }
 475
 476     /// Runs `cargo test` for clippy.
 477     fn run(self, builder: &Builder<'_>) {
 478         let stage = self.stage;
 479         let host = self.host;
 480         let compiler = builder.compiler(stage, host);
 481
 482         let clippy = builder.ensure(tool::Clippy {
 483             compiler,
 484             target: self.host,
 485             extra_features: Vec::new(),
 486         });
 487         if let Some(clippy) = clippy {
 488             let mut cargo = tool::prepare_tool_cargo(builder,
 489                                                  compiler,
 490                                                  Mode::ToolRustc,
 491                                                  host,
 492                                                  "test",
 493                                                  "src/tools/clippy",
 494                                                  SourceType::Submodule,
 495                                                  &[]);
 496
 497             // clippy tests need to know about the stage sysroot
 498             cargo.env("SYSROOT", builder.sysroot(compiler));
 499             cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
 500             cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
 501             let host_libs = builder
 502                 .stage_out(compiler, Mode::ToolRustc)
 503                 .join(builder.cargo_dir());
 504             cargo.env("HOST_LIBS", host_libs);
 505             // clippy tests need to find the driver
 506             cargo.env("CLIPPY_DRIVER_PATH", clippy);
 507
 508             builder.add_rustc_lib_path(compiler, &mut cargo);
 509
 510             if try_run(builder, &mut cargo) {
 511                 builder.save_toolstate("clippy-driver", ToolState::TestPass);
 512             }
 513         } else {
 514             eprintln!("failed to test clippy: could not build");
 515         }
 516     }
 517 }
 518
 519 fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString {
 520     // Configure PATH to find the right rustc. NB. we have to use PATH
 521     // and not RUSTC because the Cargo test suite has tests that will
 522     // fail if rustc is not spelled `rustc`.
 523     let path = builder.sysroot(compiler).join("bin");
 524     let old_path = env::var_os("PATH").unwrap_or_default();
 525     env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("")
 526 }
 527
 528 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 529 pub struct RustdocTheme {
 530     pub compiler: Compiler,
 531 }
 532
 533 impl Step for RustdocTheme {
 534     type Output = ();
 535     const DEFAULT: bool = true;
 536     const ONLY_HOSTS: bool = true;
 537
 538     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 539         run.path("src/tools/rustdoc-themes")
 540     }
 541
 542     fn make_run(run: RunConfig<'_>) {
 543         let compiler = run.builder.compiler(run.builder.top_stage, run.host);
 544
 545         run.builder.ensure(RustdocTheme { compiler });
 546     }
 547
 548     fn run(self, builder: &Builder<'_>) {
 549         let rustdoc = builder.out.join("bootstrap/debug/rustdoc");
 550         let mut cmd = builder.tool_cmd(Tool::RustdocTheme);
 551         cmd.arg(rustdoc.to_str().unwrap())
 552             .arg(
 553                 builder
 554                     .src
 555                     .join("src/librustdoc/html/static/themes")
 556                     .to_str()
 557                     .unwrap(),
 558             )
 559             .env("RUSTC_STAGE", self.compiler.stage.to_string())
 560             .env("RUSTC_SYSROOT", builder.sysroot(self.compiler))
 561             .env(
 562                 "RUSTDOC_LIBDIR",
 563                 builder.sysroot_libdir(self.compiler, self.compiler.host),
 564             )
 565             .env("CFG_RELEASE_CHANNEL", &builder.config.channel)
 566             .env("RUSTDOC_REAL", builder.rustdoc(self.compiler))
 567             .env("RUSTDOC_CRATE_VERSION", builder.rust_version())
 568             .env("RUSTC_BOOTSTRAP", "1");
 569         if let Some(linker) = builder.linker(self.compiler.host) {
 570             cmd.env("RUSTC_TARGET_LINKER", linker);
 571         }
 572         try_run(builder, &mut cmd);
 573     }
 574 }
 575
 576 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 577 pub struct RustdocJSStd {
 578     pub host: Interned<String>,
 579     pub target: Interned<String>,
 580 }
 581
 582 impl Step for RustdocJSStd {
 583     type Output = ();
 584     const DEFAULT: bool = true;
 585     const ONLY_HOSTS: bool = true;
 586
 587     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 588         run.path("src/test/rustdoc-js-std")
 589     }
 590
 591     fn make_run(run: RunConfig<'_>) {
 592         run.builder.ensure(RustdocJSStd {
 593             host: run.host,
 594             target: run.target,
 595         });
 596     }
 597
 598     fn run(self, builder: &Builder<'_>) {
 599         if let Some(ref nodejs) = builder.config.nodejs {
 600             let mut command = Command::new(nodejs);
 601             command.args(&["src/tools/rustdoc-js-std/tester.js", &*self.host]);
 602             builder.ensure(crate::doc::Std {
 603                 target: self.target,
 604                 stage: builder.top_stage,
 605             });
 606             builder.run(&mut command);
 607         } else {
 608             builder.info(
 609                 "No nodejs found, skipping \"src/test/rustdoc-js-std\" tests"
 610             );
 611         }
 612     }
 613 }
 614
 615 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 616 pub struct RustdocJSNotStd {
 617     pub host: Interned<String>,
 618     pub target: Interned<String>,
 619     pub compiler: Compiler,
 620 }
 621
 622 impl Step for RustdocJSNotStd {
 623     type Output = ();
 624     const DEFAULT: bool = true;
 625     const ONLY_HOSTS: bool = true;
 626
 627     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 628         run.path("src/test/rustdoc-js")
 629     }
 630
 631     fn make_run(run: RunConfig<'_>) {
 632         let compiler = run.builder.compiler(run.builder.top_stage, run.host);
 633         run.builder.ensure(RustdocJSNotStd {
 634             host: run.host,
 635             target: run.target,
 636             compiler,
 637         });
 638     }
 639
 640     fn run(self, builder: &Builder<'_>) {
 641         if builder.config.nodejs.is_some() {
 642             builder.ensure(Compiletest {
 643                 compiler: self.compiler,
 644                 target: self.target,
 645                 mode: "js-doc-test",
 646                 suite: "rustdoc-js",
 647                 path: None,
 648                 compare_mode: None,
 649             });
 650         } else {
 651             builder.info(
 652                 "No nodejs found, skipping \"src/test/rustdoc-js\" tests"
 653             );
 654         }
 655     }
 656 }
 657
 658 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 659 pub struct RustdocUi {
 660     pub host: Interned<String>,
 661     pub target: Interned<String>,
 662     pub compiler: Compiler,
 663 }
 664
 665 impl Step for RustdocUi {
 666     type Output = ();
 667     const DEFAULT: bool = true;
 668     const ONLY_HOSTS: bool = true;
 669
 670     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 671         run.path("src/test/rustdoc-ui")
 672     }
 673
 674     fn make_run(run: RunConfig<'_>) {
 675         let compiler = run.builder.compiler(run.builder.top_stage, run.host);
 676         run.builder.ensure(RustdocUi {
 677             host: run.host,
 678             target: run.target,
 679             compiler,
 680         });
 681     }
 682
 683     fn run(self, builder: &Builder<'_>) {
 684         builder.ensure(Compiletest {
 685             compiler: self.compiler,
 686             target: self.target,
 687             mode: "ui",
 688             suite: "rustdoc-ui",
 689             path: Some("src/test/rustdoc-ui"),
 690             compare_mode: None,
 691         })
 692     }
 693 }
 694
 695 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 696 pub struct Tidy;
 697
 698 impl Step for Tidy {
 699     type Output = ();
 700     const DEFAULT: bool = true;
 701     const ONLY_HOSTS: bool = true;
 702
 703     /// Runs the `tidy` tool.
 704     ///
 705     /// This tool in `src/tools` checks up on various bits and pieces of style and
 706     /// otherwise just implements a few lint-like checks that are specific to the
 707     /// compiler itself.
 708     fn run(self, builder: &Builder<'_>) {
 709         let mut cmd = builder.tool_cmd(Tool::Tidy);
 710         cmd.arg(builder.src.join("src"));
 711         cmd.arg(&builder.initial_cargo);
 712         if !builder.config.vendor {
 713             cmd.arg("--no-vendor");
 714         }
 715         if builder.is_verbose() {
 716             cmd.arg("--verbose");
 717         }
 718
 719         builder.info("tidy check");
 720         try_run(builder, &mut cmd);
 721     }
 722
 723     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 724         run.path("src/tools/tidy")
 725     }
 726
 727     fn make_run(run: RunConfig<'_>) {
 728         run.builder.ensure(Tidy);
 729     }
 730 }
 731
 732 fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf {
 733     builder.out.join(host).join("test")
 734 }
 735
 736 macro_rules! default_test {
 737     ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => {
 738         test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false });
 739     }
 740 }
 741
 742 macro_rules! default_test_with_compare_mode {
 743     ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr,
 744                    compare_mode: $compare_mode:expr }) => {
 745         test_with_compare_mode!($name { path: $path, mode: $mode, suite: $suite, default: true,
 746                                         host: false, compare_mode: $compare_mode });
 747     }
 748 }
 749
 750 macro_rules! host_test {
 751     ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => {
 752         test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true });
 753     }
 754 }
 755
 756 macro_rules! test {
 757     ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr,
 758                    host: $host:expr }) => {
 759         test_definitions!($name { path: $path, mode: $mode, suite: $suite, default: $default,
 760                                   host: $host, compare_mode: None });
 761     }
 762 }
 763
 764 macro_rules! test_with_compare_mode {
 765     ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr,
 766                    host: $host:expr, compare_mode: $compare_mode:expr }) => {
 767         test_definitions!($name { path: $path, mode: $mode, suite: $suite, default: $default,
 768                                   host: $host, compare_mode: Some($compare_mode) });
 769     }
 770 }
 771
 772 macro_rules! test_definitions {
 773     ($name:ident {
 774         path: $path:expr,
 775         mode: $mode:expr,
 776         suite: $suite:expr,
 777         default: $default:expr,
 778         host: $host:expr,
 779         compare_mode: $compare_mode:expr
 780     }) => {
 781         #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 782         pub struct $name {
 783             pub compiler: Compiler,
 784             pub target: Interned<String>,
 785         }
 786
 787         impl Step for $name {
 788             type Output = ();
 789             const DEFAULT: bool = $default;
 790             const ONLY_HOSTS: bool = $host;
 791
 792             fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 793                 run.suite_path($path)
 794             }
 795
 796             fn make_run(run: RunConfig<'_>) {
 797                 let compiler = run.builder.compiler(run.builder.top_stage, run.host);
 798
 799                 run.builder.ensure($name {
 800                     compiler,
 801                     target: run.target,
 802                 });
 803             }
 804
 805             fn run(self, builder: &Builder<'_>) {
 806                 builder.ensure(Compiletest {
 807                     compiler: self.compiler,
 808                     target: self.target,
 809                     mode: $mode,
 810                     suite: $suite,
 811                     path: Some($path),
 812                     compare_mode: $compare_mode,
 813                 })
 814             }
 815         }
 816     }
 817 }
 818
 819 default_test_with_compare_mode!(Ui {
 820     path: "src/test/ui",
 821     mode: "ui",
 822     suite: "ui",
 823     compare_mode: "nll"
 824 });
 825
 826 default_test!(CompileFail {
 827     path: "src/test/compile-fail",
 828     mode: "compile-fail",
 829     suite: "compile-fail"
 830 });
 831
 832 default_test!(RunFail {
 833     path: "src/test/run-fail",
 834     mode: "run-fail",
 835     suite: "run-fail"
 836 });
 837
 838 default_test!(RunPassValgrind {
 839     path: "src/test/run-pass-valgrind",
 840     mode: "run-pass-valgrind",
 841     suite: "run-pass-valgrind"
 842 });
 843
 844 default_test!(MirOpt {
 845     path: "src/test/mir-opt",
 846     mode: "mir-opt",
 847     suite: "mir-opt"
 848 });
 849
 850 default_test!(Codegen {
 851     path: "src/test/codegen",
 852     mode: "codegen",
 853     suite: "codegen"
 854 });
 855
 856 default_test!(CodegenUnits {
 857     path: "src/test/codegen-units",
 858     mode: "codegen-units",
 859     suite: "codegen-units"
 860 });
 861
 862 default_test!(Incremental {
 863     path: "src/test/incremental",
 864     mode: "incremental",
 865     suite: "incremental"
 866 });
 867
 868 default_test!(Debuginfo {
 869     path: "src/test/debuginfo",
 870     mode: "debuginfo",
 871     suite: "debuginfo"
 872 });
 873
 874 host_test!(UiFullDeps {
 875     path: "src/test/ui-fulldeps",
 876     mode: "ui",
 877     suite: "ui-fulldeps"
 878 });
 879
 880 host_test!(Rustdoc {
 881     path: "src/test/rustdoc",
 882     mode: "rustdoc",
 883     suite: "rustdoc"
 884 });
 885
 886 host_test!(Pretty {
 887     path: "src/test/pretty",
 888     mode: "pretty",
 889     suite: "pretty"
 890 });
 891 test!(RunFailPretty {
 892     path: "src/test/run-fail/pretty",
 893     mode: "pretty",
 894     suite: "run-fail",
 895     default: false,
 896     host: true
 897 });
 898 test!(RunPassValgrindPretty {
 899     path: "src/test/run-pass-valgrind/pretty",
 900     mode: "pretty",
 901     suite: "run-pass-valgrind",
 902     default: false,
 903     host: true
 904 });
 905
 906 default_test!(RunMake {
 907     path: "src/test/run-make",
 908     mode: "run-make",
 909     suite: "run-make"
 910 });
 911
 912 host_test!(RunMakeFullDeps {
 913     path: "src/test/run-make-fulldeps",
 914     mode: "run-make",
 915     suite: "run-make-fulldeps"
 916 });
 917
 918 default_test!(Assembly {
 919     path: "src/test/assembly",
 920     mode: "assembly",
 921     suite: "assembly"
 922 });
 923
 924 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 925 struct Compiletest {
 926     compiler: Compiler,
 927     target: Interned<String>,
 928     mode: &'static str,
 929     suite: &'static str,
 930     path: Option<&'static str>,
 931     compare_mode: Option<&'static str>,
 932 }
 933
 934 impl Step for Compiletest {
 935     type Output = ();
 936
 937     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
 938         run.never()
 939     }
 940
 941     /// Executes the `compiletest` tool to run a suite of tests.
 942     ///
 943     /// Compiles all tests with `compiler` for `target` with the specified
 944     /// compiletest `mode` and `suite` arguments. For example `mode` can be
 945     /// "run-pass" or `suite` can be something like `debuginfo`.
 946     fn run(self, builder: &Builder<'_>) {
 947         let compiler = self.compiler;
 948         let target = self.target;
 949         let mode = self.mode;
 950         let suite = self.suite;
 951
 952         // Path for test suite
 953         let suite_path = self.path.unwrap_or("");
 954
 955         // Skip codegen tests if they aren't enabled in configuration.
 956         if !builder.config.codegen_tests && suite == "codegen" {
 957             return;
 958         }
 959
 960         if suite == "debuginfo" {
 961             let msvc = builder.config.build.contains("msvc");
 962             if mode == "debuginfo" {
 963                 return builder.ensure(Compiletest {
 964                     mode: if msvc { "debuginfo-cdb" } else { "debuginfo-gdb+lldb" },
 965                     ..self
 966                 });
 967             }
 968
 969             builder.ensure(dist::DebuggerScripts {
 970                 sysroot: builder.sysroot(compiler),
 971                 host: target,
 972             });
 973         }
 974
 975         if suite.ends_with("fulldeps") {
 976             builder.ensure(compile::Rustc { compiler, target });
 977         }
 978
 979         if builder.no_std(target) == Some(true) {
 980             // the `test` doesn't compile for no-std targets
 981             builder.ensure(compile::Std { compiler, target });
 982         } else {
 983             builder.ensure(compile::Test { compiler, target });
 984         }
 985
 986         if builder.no_std(target) == Some(true) {
 987             // for no_std run-make (e.g., thumb*),
 988             // we need a host compiler which is called by cargo.
 989             builder.ensure(compile::Std { compiler, target: compiler.host });
 990         }
 991
 992         // HACK(eddyb) ensure that `libproc_macro` is available on the host.
 993         builder.ensure(compile::Test { compiler, target: compiler.host });
 994         // Also provide `rust_test_helpers` for the host.
 995         builder.ensure(native::TestHelpers { target: compiler.host });
 996
 997         // wasm32 can't build the test helpers
 998         if !target.contains("wasm32") {
 999             builder.ensure(native::TestHelpers { target });
1000         }
1001         builder.ensure(RemoteCopyLibs { compiler, target });
1002
1003         let mut cmd = builder.tool_cmd(Tool::Compiletest);
1004
1005         // compiletest currently has... a lot of arguments, so let's just pass all
1006         // of them!
1007
1008         cmd.arg("--compile-lib-path")
1009             .arg(builder.rustc_libdir(compiler));
1010         cmd.arg("--run-lib-path")
1011             .arg(builder.sysroot_libdir(compiler, target));
1012         cmd.arg("--rustc-path").arg(builder.rustc(compiler));
1013
1014         let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js");
1015
1016         // Avoid depending on rustdoc when we don't need it.
1017         if mode == "rustdoc"
1018             || (mode == "run-make" && suite.ends_with("fulldeps"))
1019             || (mode == "ui" && is_rustdoc)
1020             || mode == "js-doc-test"
1021         {
1022             cmd.arg("--rustdoc-path")
1023                 .arg(builder.rustdoc(compiler));
1024         }
1025
1026         cmd.arg("--src-base")
1027             .arg(builder.src.join("src/test").join(suite));
1028         cmd.arg("--build-base")
1029             .arg(testdir(builder, compiler.host).join(suite));
1030         cmd.arg("--stage-id")
1031             .arg(format!("stage{}-{}", compiler.stage, target));
1032         cmd.arg("--mode").arg(mode);
1033         cmd.arg("--target").arg(target);
1034         cmd.arg("--host").arg(&*compiler.host);
1035         cmd.arg("--llvm-filecheck")
1036             .arg(builder.llvm_filecheck(builder.config.build));
1037
1038         if builder.config.cmd.bless() {
1039             cmd.arg("--bless");
1040         }
1041
1042         let compare_mode = builder.config.cmd.compare_mode().or_else(|| {
1043             if builder.config.test_compare_mode {
1044                 self.compare_mode
1045             } else {
1046                 None
1047             }
1048         });
1049
1050         if let Some(ref pass) = builder.config.cmd.pass() {
1051             cmd.arg("--pass");
1052             cmd.arg(pass);
1053         }
1054
1055         if let Some(ref nodejs) = builder.config.nodejs {
1056             cmd.arg("--nodejs").arg(nodejs);
1057         }
1058
1059         let mut flags = if is_rustdoc {
1060             Vec::new()
1061         } else {
1062             vec!["-Crpath".to_string()]
1063         };
1064         if !is_rustdoc {
1065             if builder.config.rust_optimize_tests {
1066                 flags.push("-O".to_string());
1067             }
1068         }
1069         flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests));
1070         flags.push("-Zunstable-options".to_string());
1071         flags.push(builder.config.cmd.rustc_args().join(" "));
1072
1073         if let Some(linker) = builder.linker(target) {
1074             cmd.arg("--linker").arg(linker);
1075         }
1076
1077         let mut hostflags = flags.clone();
1078         hostflags.push(format!(
1079             "-Lnative={}",
1080             builder.test_helpers_out(compiler.host).display()
1081         ));
1082         cmd.arg("--host-rustcflags").arg(hostflags.join(" "));
1083
1084         let mut targetflags = flags;
1085         targetflags.push(format!(
1086             "-Lnative={}",
1087             builder.test_helpers_out(target).display()
1088         ));
1089         cmd.arg("--target-rustcflags").arg(targetflags.join(" "));
1090
1091         cmd.arg("--docck-python").arg(builder.python());
1092
1093         if builder.config.build.ends_with("apple-darwin") {
1094             // Force /usr/bin/python on macOS for LLDB tests because we're loading the
1095             // LLDB plugin's compiled module which only works with the system python
1096             // (namely not Homebrew-installed python)
1097             cmd.arg("--lldb-python").arg("/usr/bin/python");
1098         } else {
1099             cmd.arg("--lldb-python").arg(builder.python());
1100         }
1101
1102         if let Some(ref gdb) = builder.config.gdb {
1103             cmd.arg("--gdb").arg(gdb);
1104         }
1105
1106         let run = |cmd: &mut Command| {
1107             cmd.output().map(|output| {
1108                 String::from_utf8_lossy(&output.stdout)
1109                     .lines().next().unwrap_or_else(|| {
1110                         panic!("{:?} failed {:?}", cmd, output)
1111                     }).to_string()
1112             })
1113         };
1114         let lldb_exe = if builder.config.lldb_enabled && !target.contains("emscripten") {
1115             // Test against the lldb that was just built.
1116             builder.llvm_out(target).join("bin").join("lldb")
1117         } else {
1118             PathBuf::from("lldb")
1119         };
1120         let lldb_version = Command::new(&lldb_exe)
1121             .arg("--version")
1122             .output()
1123             .map(|output| { String::from_utf8_lossy(&output.stdout).to_string() })
1124             .ok();
1125         if let Some(ref vers) = lldb_version {
1126             cmd.arg("--lldb-version").arg(vers);
1127             let lldb_python_dir = run(Command::new(&lldb_exe).arg("-P")).ok();
1128             if let Some(ref dir) = lldb_python_dir {
1129                 cmd.arg("--lldb-python-dir").arg(dir);
1130             }
1131         }
1132
1133         if util::forcing_clang_based_tests() {
1134             let clang_exe = builder.llvm_out(target).join("bin").join("clang");
1135             cmd.arg("--run-clang-based-tests-with").arg(clang_exe);
1136         }
1137
1138         // Get paths from cmd args
1139         let paths = match &builder.config.cmd {
1140             Subcommand::Test { ref paths, .. } => &paths[..],
1141             _ => &[],
1142         };
1143
1144         // Get test-args by striping suite path
1145         let mut test_args: Vec<&str> = paths
1146             .iter()
1147             .map(|p| {
1148                 match p.strip_prefix(".") {
1149                     Ok(path) => path,
1150                     Err(_) => p,
1151                 }
1152             })
1153             .filter(|p| p.starts_with(suite_path) && (p.is_dir() || p.is_file()))
1154             .filter_map(|p| {
1155                 // Since test suite paths are themselves directories, if we don't
1156                 // specify a directory or file, we'll get an empty string here
1157                 // (the result of the test suite directory without its suite prefix).
1158                 // Therefore, we need to filter these out, as only the first --test-args
1159                 // flag is respected, so providing an empty --test-args conflicts with
1160                 // any following it.
1161                 match p.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) {
1162                     Some(s) if s != "" => Some(s),
1163                     _ => None,
1164                 }
1165             })
1166             .collect();
1167
1168         test_args.append(&mut builder.config.cmd.test_args());
1169
1170         cmd.args(&test_args);
1171
1172         if builder.is_verbose() {
1173             cmd.arg("--verbose");
1174         }
1175
1176         if !builder.config.verbose_tests {
1177             cmd.arg("--quiet");
1178         }
1179
1180         if builder.config.llvm_enabled() {
1181             let llvm_config = builder.ensure(native::Llvm {
1182                 target: builder.config.build,
1183                 emscripten: false,
1184             });
1185             if !builder.config.dry_run {
1186                 let llvm_version = output(Command::new(&llvm_config).arg("--version"));
1187                 cmd.arg("--llvm-version").arg(llvm_version);
1188             }
1189             if !builder.is_rust_llvm(target) {
1190                 cmd.arg("--system-llvm");
1191             }
1192
1193             // Only pass correct values for these flags for the `run-make` suite as it
1194             // requires that a C++ compiler was configured which isn't always the case.
1195             if !builder.config.dry_run && suite == "run-make-fulldeps" {
1196                 let llvm_components = output(Command::new(&llvm_config).arg("--components"));
1197                 let llvm_cxxflags = output(Command::new(&llvm_config).arg("--cxxflags"));
1198                 cmd.arg("--cc")
1199                     .arg(builder.cc(target))
1200                     .arg("--cxx")
1201                     .arg(builder.cxx(target).unwrap())
1202                     .arg("--cflags")
1203                     .arg(builder.cflags(target, GitRepo::Rustc).join(" "))
1204                     .arg("--llvm-components")
1205                     .arg(llvm_components.trim())
1206                     .arg("--llvm-cxxflags")
1207                     .arg(llvm_cxxflags.trim());
1208                 if let Some(ar) = builder.ar(target) {
1209                     cmd.arg("--ar").arg(ar);
1210                 }
1211
1212                 // The llvm/bin directory contains many useful cross-platform
1213                 // tools. Pass the path to run-make tests so they can use them.
1214                 let llvm_bin_path = llvm_config.parent()
1215                     .expect("Expected llvm-config to be contained in directory");
1216                 assert!(llvm_bin_path.is_dir());
1217                 cmd.arg("--llvm-bin-dir").arg(llvm_bin_path);
1218
1219                 // If LLD is available, add it to the PATH
1220                 if builder.config.lld_enabled {
1221                     let lld_install_root = builder.ensure(native::Lld {
1222                         target: builder.config.build,
1223                     });
1224
1225                     let lld_bin_path = lld_install_root.join("bin");
1226
1227                     let old_path = env::var_os("PATH").unwrap_or_default();
1228                     let new_path = env::join_paths(std::iter::once(lld_bin_path)
1229                         .chain(env::split_paths(&old_path)))
1230                         .expect("Could not add LLD bin path to PATH");
1231                     cmd.env("PATH", new_path);
1232                 }
1233             }
1234         }
1235
1236         if suite != "run-make-fulldeps" {
1237             cmd.arg("--cc")
1238                 .arg("")
1239                 .arg("--cxx")
1240                 .arg("")
1241                 .arg("--cflags")
1242                 .arg("")
1243                 .arg("--llvm-components")
1244                 .arg("")
1245                 .arg("--llvm-cxxflags")
1246                 .arg("");
1247         }
1248
1249         if builder.remote_tested(target) {
1250             cmd.arg("--remote-test-client")
1251                 .arg(builder.tool_exe(Tool::RemoteTestClient));
1252         }
1253
1254         // Running a C compiler on MSVC requires a few env vars to be set, to be
1255         // sure to set them here.
1256         //
1257         // Note that if we encounter `PATH` we make sure to append to our own `PATH`
1258         // rather than stomp over it.
1259         if target.contains("msvc") {
1260             for &(ref k, ref v) in builder.cc[&target].env() {
1261                 if k != "PATH" {
1262                     cmd.env(k, v);
1263                 }
1264             }
1265         }
1266         cmd.env("RUSTC_BOOTSTRAP", "1");
1267         builder.add_rust_test_threads(&mut cmd);
1268
1269         if builder.config.sanitizers {
1270             cmd.env("RUSTC_SANITIZER_SUPPORT", "1");
1271         }
1272
1273         if builder.config.profiler {
1274             cmd.env("RUSTC_PROFILER_SUPPORT", "1");
1275         }
1276
1277         cmd.env("RUST_TEST_TMPDIR", builder.out.join("tmp"));
1278
1279         cmd.arg("--adb-path").arg("adb");
1280         cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR);
1281         if target.contains("android") {
1282             // Assume that cc for this target comes from the android sysroot
1283             cmd.arg("--android-cross-path")
1284                 .arg(builder.cc(target).parent().unwrap().parent().unwrap());
1285         } else {
1286             cmd.arg("--android-cross-path").arg("");
1287         }
1288
1289         if builder.config.cmd.rustfix_coverage() {
1290             cmd.arg("--rustfix-coverage");
1291         }
1292
1293         builder.ci_env.force_coloring_in_ci(&mut cmd);
1294
1295         builder.info(&format!(
1296             "Check compiletest suite={} mode={} ({} -> {})",
1297             suite, mode, &compiler.host, target
1298         ));
1299         let _time = util::timeit(&builder);
1300         try_run(builder, &mut cmd);
1301
1302         if let Some(compare_mode) = compare_mode {
1303             cmd.arg("--compare-mode").arg(compare_mode);
1304             builder.info(&format!(
1305                 "Check compiletest suite={} mode={} compare_mode={} ({} -> {})",
1306                 suite, mode, compare_mode, &compiler.host, target
1307             ));
1308             let _time = util::timeit(&builder);
1309             try_run(builder, &mut cmd);
1310         }
1311     }
1312 }
1313
1314 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1315 struct DocTest {
1316     compiler: Compiler,
1317     path: &'static str,
1318     name: &'static str,
1319     is_ext_doc: bool,
1320 }
1321
1322 impl Step for DocTest {
1323     type Output = ();
1324     const ONLY_HOSTS: bool = true;
1325
1326     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1327         run.never()
1328     }
1329
1330     /// Runs `rustdoc --test` for all documentation in `src/doc`.
1331     ///
1332     /// This will run all tests in our markdown documentation (e.g., the book)
1333     /// located in `src/doc`. The `rustdoc` that's run is the one that sits next to
1334     /// `compiler`.
1335     fn run(self, builder: &Builder<'_>) {
1336         let compiler = self.compiler;
1337
1338         builder.ensure(compile::Test {
1339             compiler,
1340             target: compiler.host,
1341         });
1342
1343         // Do a breadth-first traversal of the `src/doc` directory and just run
1344         // tests for all files that end in `*.md`
1345         let mut stack = vec![builder.src.join(self.path)];
1346         let _time = util::timeit(&builder);
1347
1348         let mut files = Vec::new();
1349         while let Some(p) = stack.pop() {
1350             if p.is_dir() {
1351                 stack.extend(t!(p.read_dir()).map(|p| t!(p).path()));
1352                 continue;
1353             }
1354
1355             if p.extension().and_then(|s| s.to_str()) != Some("md") {
1356                 continue;
1357             }
1358
1359             // The nostarch directory in the book is for no starch, and so isn't
1360             // guaranteed to builder. We don't care if it doesn't build, so skip it.
1361             if p.to_str().map_or(false, |p| p.contains("nostarch")) {
1362                 continue;
1363             }
1364
1365             files.push(p);
1366         }
1367
1368         files.sort();
1369
1370         let mut toolstate = ToolState::TestPass;
1371         for file in files {
1372             if !markdown_test(builder, compiler, &file) {
1373                 toolstate = ToolState::TestFail;
1374             }
1375         }
1376         if self.is_ext_doc {
1377             builder.save_toolstate(self.name, toolstate);
1378         }
1379     }
1380 }
1381
1382 macro_rules! test_book {
1383     ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => {
1384         $(
1385             #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1386             pub struct $name {
1387                 compiler: Compiler,
1388             }
1389
1390             impl Step for $name {
1391                 type Output = ();
1392                 const DEFAULT: bool = $default;
1393                 const ONLY_HOSTS: bool = true;
1394
1395                 fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1396                     run.path($path)
1397                 }
1398
1399                 fn make_run(run: RunConfig<'_>) {
1400                     run.builder.ensure($name {
1401                         compiler: run.builder.compiler(run.builder.top_stage, run.host),
1402                     });
1403                 }
1404
1405                 fn run(self, builder: &Builder<'_>) {
1406                     builder.ensure(DocTest {
1407                         compiler: self.compiler,
1408                         path: $path,
1409                         name: $book_name,
1410                         is_ext_doc: !$default,
1411                     });
1412                 }
1413             }
1414         )+
1415     }
1416 }
1417
1418 test_book!(
1419     Nomicon, "src/doc/nomicon", "nomicon", default=false;
1420     Reference, "src/doc/reference", "reference", default=false;
1421     RustdocBook, "src/doc/rustdoc", "rustdoc", default=true;
1422     RustcBook, "src/doc/rustc", "rustc", default=true;
1423     RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false;
1424     EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false;
1425     TheBook, "src/doc/book", "book", default=false;
1426     UnstableBook, "src/doc/unstable-book", "unstable-book", default=true;
1427     EditionGuide, "src/doc/edition-guide", "edition-guide", default=false;
1428 );
1429
1430 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1431 pub struct ErrorIndex {
1432     compiler: Compiler,
1433 }
1434
1435 impl Step for ErrorIndex {
1436     type Output = ();
1437     const DEFAULT: bool = true;
1438     const ONLY_HOSTS: bool = true;
1439
1440     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1441         run.path("src/tools/error_index_generator")
1442     }
1443
1444     fn make_run(run: RunConfig<'_>) {
1445         run.builder.ensure(ErrorIndex {
1446             compiler: run.builder.compiler(run.builder.top_stage, run.host),
1447         });
1448     }
1449
1450     /// Runs the error index generator tool to execute the tests located in the error
1451     /// index.
1452     ///
1453     /// The `error_index_generator` tool lives in `src/tools` and is used to
1454     /// generate a markdown file from the error indexes of the code base which is
1455     /// then passed to `rustdoc --test`.
1456     fn run(self, builder: &Builder<'_>) {
1457         let compiler = self.compiler;
1458
1459         builder.ensure(compile::Std {
1460             compiler,
1461             target: compiler.host,
1462         });
1463
1464         let dir = testdir(builder, compiler.host);
1465         t!(fs::create_dir_all(&dir));
1466         let output = dir.join("error-index.md");
1467
1468         let mut tool = tool::ErrorIndex::command(
1469             builder,
1470             builder.compiler(compiler.stage, builder.config.build),
1471         );
1472         tool.arg("markdown")
1473             .arg(&output)
1474             .env("CFG_BUILD", &builder.config.build)
1475             .env("RUSTC_ERROR_METADATA_DST", builder.extended_error_dir());
1476
1477         builder.info(&format!("Testing error-index stage{}", compiler.stage));
1478         let _time = util::timeit(&builder);
1479         builder.run_quiet(&mut tool);
1480         markdown_test(builder, compiler, &output);
1481     }
1482 }
1483
1484 fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool {
1485     match fs::read_to_string(markdown) {
1486         Ok(contents) => {
1487             if !contents.contains("```") {
1488                 return true;
1489             }
1490         }
1491         Err(_) => {}
1492     }
1493
1494     builder.info(&format!("doc tests for: {}", markdown.display()));
1495     let mut cmd = builder.rustdoc_cmd(compiler);
1496     builder.add_rust_test_threads(&mut cmd);
1497     cmd.arg("--test");
1498     cmd.arg(markdown);
1499     cmd.env("RUSTC_BOOTSTRAP", "1");
1500
1501     let test_args = builder.config.cmd.test_args().join(" ");
1502     cmd.arg("--test-args").arg(test_args);
1503
1504     if builder.config.verbose_tests {
1505         try_run(builder, &mut cmd)
1506     } else {
1507         try_run_quiet(builder, &mut cmd)
1508     }
1509 }
1510
1511 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1512 pub struct RustcGuide;
1513
1514 impl Step for RustcGuide {
1515     type Output = ();
1516     const DEFAULT: bool = false;
1517     const ONLY_HOSTS: bool = true;
1518
1519     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1520         run.path("src/doc/rustc-guide")
1521     }
1522
1523     fn make_run(run: RunConfig<'_>) {
1524         run.builder.ensure(RustcGuide);
1525     }
1526
1527     fn run(self, builder: &Builder<'_>) {
1528         let src = builder.src.join("src/doc/rustc-guide");
1529         let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
1530         let toolstate = if try_run(builder, rustbook_cmd.arg("linkcheck").arg(&src)) {
1531             ToolState::TestPass
1532         } else {
1533             ToolState::TestFail
1534         };
1535         builder.save_toolstate("rustc-guide", toolstate);
1536     }
1537 }
1538
1539 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1540 pub struct CrateLibrustc {
1541     compiler: Compiler,
1542     target: Interned<String>,
1543     test_kind: TestKind,
1544     krate: Interned<String>,
1545 }
1546
1547 impl Step for CrateLibrustc {
1548     type Output = ();
1549     const DEFAULT: bool = true;
1550     const ONLY_HOSTS: bool = true;
1551
1552     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1553         run.krate("rustc-main")
1554     }
1555
1556     fn make_run(run: RunConfig<'_>) {
1557         let builder = run.builder;
1558         let compiler = builder.compiler(builder.top_stage, run.host);
1559
1560         for krate in builder.in_tree_crates("rustc-main") {
1561             if run.path.ends_with(&krate.path) {
1562                 let test_kind = builder.kind.into();
1563
1564                 builder.ensure(CrateLibrustc {
1565                     compiler,
1566                     target: run.target,
1567                     test_kind,
1568                     krate: krate.name,
1569                 });
1570             }
1571         }
1572     }
1573
1574     fn run(self, builder: &Builder<'_>) {
1575         builder.ensure(Crate {
1576             compiler: self.compiler,
1577             target: self.target,
1578             mode: Mode::Rustc,
1579             test_kind: self.test_kind,
1580             krate: self.krate,
1581         });
1582     }
1583 }
1584
1585 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1586 pub struct CrateNotDefault {
1587     compiler: Compiler,
1588     target: Interned<String>,
1589     test_kind: TestKind,
1590     krate: &'static str,
1591 }
1592
1593 impl Step for CrateNotDefault {
1594     type Output = ();
1595
1596     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1597         run.path("src/librustc_asan")
1598             .path("src/librustc_lsan")
1599             .path("src/librustc_msan")
1600             .path("src/librustc_tsan")
1601     }
1602
1603     fn make_run(run: RunConfig<'_>) {
1604         let builder = run.builder;
1605         let compiler = builder.compiler(builder.top_stage, run.host);
1606
1607         let test_kind = builder.kind.into();
1608
1609         builder.ensure(CrateNotDefault {
1610             compiler,
1611             target: run.target,
1612             test_kind,
1613             krate: match run.path {
1614                 _ if run.path.ends_with("src/librustc_asan") => "rustc_asan",
1615                 _ if run.path.ends_with("src/librustc_lsan") => "rustc_lsan",
1616                 _ if run.path.ends_with("src/librustc_msan") => "rustc_msan",
1617                 _ if run.path.ends_with("src/librustc_tsan") => "rustc_tsan",
1618                 _ => panic!("unexpected path {:?}", run.path),
1619             },
1620         });
1621     }
1622
1623     fn run(self, builder: &Builder<'_>) {
1624         builder.ensure(Crate {
1625             compiler: self.compiler,
1626             target: self.target,
1627             mode: Mode::Std,
1628             test_kind: self.test_kind,
1629             krate: INTERNER.intern_str(self.krate),
1630         });
1631     }
1632 }
1633
1634 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
1635 pub struct Crate {
1636     pub compiler: Compiler,
1637     pub target: Interned<String>,
1638     pub mode: Mode,
1639     pub test_kind: TestKind,
1640     pub krate: Interned<String>,
1641 }
1642
1643 impl Step for Crate {
1644     type Output = ();
1645     const DEFAULT: bool = true;
1646
1647     fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> {
1648         let builder = run.builder;
1649         run = run.krate("test");
1650         for krate in run.builder.in_tree_crates("std") {
1651             if !(krate.name.starts_with("rustc_") && krate.name.ends_with("san")) {
1652                 run = run.path(krate.local_path(&builder).to_str().unwrap());
1653             }
1654         }
1655         run
1656     }
1657
1658     fn make_run(run: RunConfig<'_>) {
1659         let builder = run.builder;
1660         let compiler = builder.compiler(builder.top_stage, run.host);
1661
1662         let make = |mode: Mode, krate: &CargoCrate| {
1663             let test_kind = builder.kind.into();
1664
1665             builder.ensure(Crate {
1666                 compiler,
1667                 target: run.target,
1668                 mode,
1669                 test_kind,
1670                 krate: krate.name,
1671             });
1672         };
1673
1674         for krate in builder.in_tree_crates("std") {
1675             if run.path.ends_with(&krate.local_path(&builder)) {
1676                 make(Mode::Std, krate);
1677             }
1678         }
1679         for krate in builder.in_tree_crates("test") {
1680             if run.path.ends_with(&krate.local_path(&builder)) {
1681                 make(Mode::Test, krate);
1682             }
1683         }
1684     }
1685
1686     /// Runs all unit tests plus documentation tests for a given crate defined
1687     /// by a `Cargo.toml` (single manifest)
1688     ///
1689     /// This is what runs tests for crates like the standard library, compiler, etc.
1690     /// It essentially is the driver for running `cargo test`.
1691     ///
1692     /// Currently this runs all tests for a DAG by passing a bunch of `-p foo`
1693     /// arguments, and those arguments are discovered from `cargo metadata`.
1694     fn run(self, builder: &Builder<'_>) {
1695         let compiler = self.compiler;
1696         let target = self.target;
1697         let mode = self.mode;
1698         let test_kind = self.test_kind;
1699         let krate = self.krate;
1700
1701         builder.ensure(compile::Test { compiler, target });
1702         builder.ensure(RemoteCopyLibs { compiler, target });
1703
1704         // If we're not doing a full bootstrap but we're testing a stage2
1705         // version of libstd, then what we're actually testing is the libstd
1706         // produced in stage1. Reflect that here by updating the compiler that
1707         // we're working with automatically.
1708         let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
1709
1710         let mut cargo = builder.cargo(compiler, mode, target, test_kind.subcommand());
1711         match mode {
1712             Mode::Std => {
1713                 compile::std_cargo(builder, &compiler, target, &mut cargo);
1714             }
1715             Mode::Test => {
1716                 compile::test_cargo(builder, &compiler, target, &mut cargo);
1717             }
1718             Mode::Rustc => {
1719                 builder.ensure(compile::Rustc { compiler, target });
1720                 compile::rustc_cargo(builder, &mut cargo);
1721             }
1722             _ => panic!("can only test libraries"),
1723         };
1724
1725         // Build up the base `cargo test` command.
1726         //
1727         // Pass in some standard flags then iterate over the graph we've discovered
1728         // in `cargo metadata` with the maps above and figure out what `-p`
1729         // arguments need to get passed.
1730         if test_kind.subcommand() == "test" && !builder.fail_fast {
1731             cargo.arg("--no-fail-fast");
1732         }
1733         match builder.doc_tests {
1734             DocTests::Only => {
1735                 cargo.arg("--doc");
1736             }
1737             DocTests::No => {
1738                 cargo.args(&["--lib", "--bins", "--examples", "--tests", "--benches"]);
1739             }
1740             DocTests::Yes => {}
1741         }
1742
1743         cargo.arg("-p").arg(krate);
1744
1745         // The tests are going to run with the *target* libraries, so we need to
1746         // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent.
1747         //
1748         // Note that to run the compiler we need to run with the *host* libraries,
1749         // but our wrapper scripts arrange for that to be the case anyway.
1750         let mut dylib_path = dylib_path();
1751         dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
1752         cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
1753
1754         cargo.arg("--");
1755         cargo.args(&builder.config.cmd.test_args());
1756
1757         if !builder.config.verbose_tests {
1758             cargo.arg("--quiet");
1759         }
1760
1761         if target.contains("emscripten") {
1762             cargo.env(
1763                 format!("CARGO_TARGET_{}_RUNNER", envify(&target)),
1764                 builder
1765                     .config
1766                     .nodejs
1767                     .as_ref()
1768                     .expect("nodejs not configured"),
1769             );
1770         } else if target.starts_with("wasm32") {
1771             // Warn about running tests without the `wasm_syscall` feature enabled.
1772             // The javascript shim implements the syscall interface so that test
1773             // output can be correctly reported.
1774             if !builder.config.wasm_syscall {
1775                 builder.info(
1776                     "Libstd was built without `wasm_syscall` feature enabled: \
1777                      test output may not be visible."
1778                 );
1779             }
1780
1781             // On the wasm32-unknown-unknown target we're using LTO which is
1782             // incompatible with `-C prefer-dynamic`, so disable that here
1783             cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
1784
1785             let node = builder
1786                 .config
1787                 .nodejs
1788                 .as_ref()
1789                 .expect("nodejs not configured");
1790             let runner = format!(
1791                 "{} {}/src/etc/wasm32-shim.js",
1792                 node.display(),
1793                 builder.src.display()
1794             );
1795             cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target)), &runner);
1796         } else if builder.remote_tested(target) {
1797             cargo.env(
1798                 format!("CARGO_TARGET_{}_RUNNER", envify(&target)),
1799                 format!("{} run", builder.tool_exe(Tool::RemoteTestClient).display()),
1800             );
1801         }
1802
1803         builder.info(&format!(
1804             "{} {} stage{} ({} -> {})",
1805             test_kind, krate, compiler.stage, &compiler.host, target
1806         ));
1807         let _time = util::timeit(&builder);
1808         try_run(builder, &mut cargo);
1809     }
1810 }
1811
1812 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1813 pub struct CrateRustdoc {
1814     host: Interned<String>,
1815     test_kind: TestKind,
1816 }
1817
1818 impl Step for CrateRustdoc {
1819     type Output = ();
1820     const DEFAULT: bool = true;
1821     const ONLY_HOSTS: bool = true;
1822
1823     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1824         run.paths(&["src/librustdoc", "src/tools/rustdoc"])
1825     }
1826
1827     fn make_run(run: RunConfig<'_>) {
1828         let builder = run.builder;
1829
1830         let test_kind = builder.kind.into();
1831
1832         builder.ensure(CrateRustdoc {
1833             host: run.host,
1834             test_kind,
1835         });
1836     }
1837
1838     fn run(self, builder: &Builder<'_>) {
1839         let test_kind = self.test_kind;
1840
1841         let compiler = builder.compiler(builder.top_stage, self.host);
1842         let target = compiler.host;
1843         builder.ensure(compile::Rustc { compiler, target });
1844
1845         let mut cargo = tool::prepare_tool_cargo(builder,
1846                                                  compiler,
1847                                                  Mode::ToolRustc,
1848                                                  target,
1849                                                  test_kind.subcommand(),
1850                                                  "src/tools/rustdoc",
1851                                                  SourceType::InTree,
1852                                                  &[]);
1853         if test_kind.subcommand() == "test" && !builder.fail_fast {
1854             cargo.arg("--no-fail-fast");
1855         }
1856
1857         cargo.arg("-p").arg("rustdoc:0.0.0");
1858
1859         cargo.arg("--");
1860         cargo.args(&builder.config.cmd.test_args());
1861
1862         if self.host.contains("musl") {
1863             cargo.arg("'-Ctarget-feature=-crt-static'");
1864         }
1865
1866         if !builder.config.verbose_tests {
1867             cargo.arg("--quiet");
1868         }
1869
1870         builder.info(&format!(
1871             "{} rustdoc stage{} ({} -> {})",
1872             test_kind, compiler.stage, &compiler.host, target
1873         ));
1874         let _time = util::timeit(&builder);
1875
1876         try_run(builder, &mut cargo);
1877     }
1878 }
1879
1880 fn envify(s: &str) -> String {
1881     s.chars()
1882         .map(|c| match c {
1883             '-' => '_',
1884             c => c,
1885         })
1886         .flat_map(|c| c.to_uppercase())
1887         .collect()
1888 }
1889
1890 /// Some test suites are run inside emulators or on remote devices, and most
1891 /// of our test binaries are linked dynamically which means we need to ship
1892 /// the standard library and such to the emulator ahead of time. This step
1893 /// represents this and is a dependency of all test suites.
1894 ///
1895 /// Most of the time this is a no-op. For some steps such as shipping data to
1896 /// QEMU we have to build our own tools so we've got conditional dependencies
1897 /// on those programs as well. Note that the remote test client is built for
1898 /// the build target (us) and the server is built for the target.
1899 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1900 pub struct RemoteCopyLibs {
1901     compiler: Compiler,
1902     target: Interned<String>,
1903 }
1904
1905 impl Step for RemoteCopyLibs {
1906     type Output = ();
1907
1908     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1909         run.never()
1910     }
1911
1912     fn run(self, builder: &Builder<'_>) {
1913         let compiler = self.compiler;
1914         let target = self.target;
1915         if !builder.remote_tested(target) {
1916             return;
1917         }
1918
1919         builder.ensure(compile::Test { compiler, target });
1920
1921         builder.info(&format!("REMOTE copy libs to emulator ({})", target));
1922         t!(fs::create_dir_all(builder.out.join("tmp")));
1923
1924         let server = builder.ensure(tool::RemoteTestServer {
1925             compiler: compiler.with_stage(0),
1926             target,
1927         });
1928
1929         // Spawn the emulator and wait for it to come online
1930         let tool = builder.tool_exe(Tool::RemoteTestClient);
1931         let mut cmd = Command::new(&tool);
1932         cmd.arg("spawn-emulator")
1933             .arg(target)
1934             .arg(&server)
1935             .arg(builder.out.join("tmp"));
1936         if let Some(rootfs) = builder.qemu_rootfs(target) {
1937             cmd.arg(rootfs);
1938         }
1939         builder.run(&mut cmd);
1940
1941         // Push all our dylibs to the emulator
1942         for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) {
1943             let f = t!(f);
1944             let name = f.file_name().into_string().unwrap();
1945             if util::is_dylib(&name) {
1946                 builder.run(Command::new(&tool).arg("push").arg(f.path()));
1947             }
1948         }
1949     }
1950 }
1951
1952 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
1953 pub struct Distcheck;
1954
1955 impl Step for Distcheck {
1956     type Output = ();
1957
1958     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
1959         run.path("distcheck")
1960     }
1961
1962     fn make_run(run: RunConfig<'_>) {
1963         run.builder.ensure(Distcheck);
1964     }
1965
1966     /// Runs "distcheck", a 'make check' from a tarball
1967     fn run(self, builder: &Builder<'_>) {
1968         builder.info("Distcheck");
1969         let dir = builder.out.join("tmp").join("distcheck");
1970         let _ = fs::remove_dir_all(&dir);
1971         t!(fs::create_dir_all(&dir));
1972
1973         // Guarantee that these are built before we begin running.
1974         builder.ensure(dist::PlainSourceTarball);
1975         builder.ensure(dist::Src);
1976
1977         let mut cmd = Command::new("tar");
1978         cmd.arg("-xzf")
1979             .arg(builder.ensure(dist::PlainSourceTarball))
1980             .arg("--strip-components=1")
1981             .current_dir(&dir);
1982         builder.run(&mut cmd);
1983         builder.run(
1984             Command::new("./configure")
1985                 .args(&builder.config.configure_args)
1986                 .arg("--enable-vendor")
1987                 .current_dir(&dir),
1988         );
1989         builder.run(
1990             Command::new(build_helper::make(&builder.config.build))
1991                 .arg("check")
1992                 .current_dir(&dir),
1993         );
1994
1995         // Now make sure that rust-src has all of libstd's dependencies
1996         builder.info("Distcheck rust-src");
1997         let dir = builder.out.join("tmp").join("distcheck-src");
1998         let _ = fs::remove_dir_all(&dir);
1999         t!(fs::create_dir_all(&dir));
2000
2001         let mut cmd = Command::new("tar");
2002         cmd.arg("-xzf")
2003             .arg(builder.ensure(dist::Src))
2004             .arg("--strip-components=1")
2005             .current_dir(&dir);
2006         builder.run(&mut cmd);
2007
2008         let toml = dir.join("rust-src/lib/rustlib/src/rust/src/libstd/Cargo.toml");
2009         builder.run(
2010             Command::new(&builder.initial_cargo)
2011                 .arg("generate-lockfile")
2012                 .arg("--manifest-path")
2013                 .arg(&toml)
2014                 .current_dir(&dir),
2015         );
2016     }
2017 }
2018
2019 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
2020 pub struct Bootstrap;
2021
2022 impl Step for Bootstrap {
2023     type Output = ();
2024     const DEFAULT: bool = true;
2025     const ONLY_HOSTS: bool = true;
2026
2027     /// Tests the build system itself.
2028     fn run(self, builder: &Builder<'_>) {
2029         let mut cmd = Command::new(&builder.initial_cargo);
2030         cmd.arg("test")
2031             .current_dir(builder.src.join("src/bootstrap"))
2032             .env("RUSTFLAGS", "-Cdebuginfo=2")
2033             .env("CARGO_TARGET_DIR", builder.out.join("bootstrap"))
2034             .env("RUSTC_BOOTSTRAP", "1")
2035             .env("RUSTC", &builder.initial_rustc);
2036         if let Some(flags) = option_env!("RUSTFLAGS") {
2037             // Use the same rustc flags for testing as for "normal" compilation,
2038             // so that Cargo doesn’t recompile the entire dependency graph every time:
2039             // https://github.com/rust-lang/rust/issues/49215
2040             cmd.env("RUSTFLAGS", flags);
2041         }
2042         if !builder.fail_fast {
2043             cmd.arg("--no-fail-fast");
2044         }
2045         cmd.arg("--").args(&builder.config.cmd.test_args());
2046         // rustbuild tests are racy on directory creation so just run them one at a time.
2047         // Since there's not many this shouldn't be a problem.
2048         cmd.arg("--test-threads=1");
2049         try_run(builder, &mut cmd);
2050     }
2051
2052     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
2053         run.path("src/bootstrap")
2054     }
2055
2056     fn make_run(run: RunConfig<'_>) {
2057         run.builder.ensure(Bootstrap);
2058     }
2059 }