src/libtest/lib.rs

   1 //! Support code for rustc's built in unit-test and micro-benchmarking
   2 //! framework.
   3 //!
   4 //! Almost all user code will only be interested in `Bencher` and
   5 //! `black_box`. All other interactions (such as writing tests and
   6 //! benchmarks themselves) should be done via the `#[test]` and
   7 //! `#[bench]` attributes.
   8 //!
   9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
  10
  11 // Currently, not much of this is meant for users. It is intended to
  12 // support the simplest interface possible for representing and
  13 // running tests while providing a base that other test frameworks may
  14 // build off of.
  15
  16 // N.B., this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
  17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
  18 // cargo) to detect this crate.
  19
  20 #![crate_name = "test"]
  21 #![unstable(feature = "test", issue = "50297")]
  22 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
  23 #![feature(asm)]
  24 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
  25 #![feature(rustc_private)]
  26 #![feature(nll)]
  27 #![feature(bool_to_option)]
  28 #![feature(set_stdio)]
  29 #![feature(panic_unwind)]
  30 #![feature(staged_api)]
  31 #![feature(termination_trait_lib)]
  32 #![feature(test)]
  33
  34 // Public reexports
  35 pub use self::bench::{black_box, Bencher};
  36 pub use self::console::run_tests_console;
  37 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
  38 pub use self::types::TestName::*;
  39 pub use self::types::*;
  40 pub use self::ColorConfig::*;
  41 pub use cli::TestOpts;
  42
  43 // Module to be used by rustc to compile tests in libtest
  44 pub mod test {
  45     pub use crate::{
  46         assert_test_result,
  47         bench::Bencher,
  48         cli::{parse_opts, TestOpts},
  49         filter_tests,
  50         helpers::metrics::{Metric, MetricMap},
  51         options::{Options, RunIgnored, RunStrategy, ShouldPanic},
  52         run_test, test_main, test_main_static,
  53         test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
  54         time::{TestExecTime, TestTimeOptions},
  55         types::{
  56             DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc,
  57             TestDescAndFn, TestName, TestType,
  58         },
  59     };
  60 }
  61
  62 use std::{
  63     env, io,
  64     io::prelude::Write,
  65     panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
  66     process,
  67     process::{Command, Termination},
  68     sync::mpsc::{channel, Sender},
  69     sync::{Arc, Mutex},
  70     thread,
  71     time::{Duration, Instant},
  72 };
  73
  74 pub mod bench;
  75 mod cli;
  76 mod console;
  77 mod event;
  78 mod formatters;
  79 mod helpers;
  80 mod options;
  81 pub mod stats;
  82 mod test_result;
  83 mod time;
  84 mod types;
  85
  86 #[cfg(test)]
  87 mod tests;
  88
  89 use event::{CompletedTest, TestEvent};
  90 use helpers::concurrency::get_concurrency;
  91 use helpers::exit_code::get_exit_code;
  92 use helpers::sink::Sink;
  93 use options::{Concurrent, RunStrategy};
  94 use test_result::*;
  95 use time::TestExecTime;
  96
  97 // Process exit code to be used to indicate test failures.
  98 const ERROR_EXIT_CODE: i32 = 101;
  99
 100 const SECONDARY_TEST_INVOKER_VAR: &'static str = "__RUST_TEST_INVOKE";
 101
 102 // The default console test runner. It accepts the command line
 103 // arguments and a vector of test_descs.
 104 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
 105     let mut opts = match cli::parse_opts(args) {
 106         Some(Ok(o)) => o,
 107         Some(Err(msg)) => {
 108             eprintln!("error: {}", msg);
 109             process::exit(ERROR_EXIT_CODE);
 110         }
 111         None => return,
 112     };
 113     if let Some(options) = options {
 114         opts.options = options;
 115     }
 116     if opts.list {
 117         if let Err(e) = console::list_tests_console(&opts, tests) {
 118             eprintln!("error: io error when listing tests: {:?}", e);
 119             process::exit(ERROR_EXIT_CODE);
 120         }
 121     } else {
 122         match console::run_tests_console(&opts, tests) {
 123             Ok(true) => {}
 124             Ok(false) => process::exit(ERROR_EXIT_CODE),
 125             Err(e) => {
 126                 eprintln!("error: io error when listing tests: {:?}", e);
 127                 process::exit(ERROR_EXIT_CODE);
 128             }
 129         }
 130     }
 131 }
 132
 133 /// A variant optimized for invocation with a static test vector.
 134 /// This will panic (intentionally) when fed any dynamic tests.
 135 ///
 136 /// This is the entry point for the main function generated by `rustc --test`
 137 /// when panic=unwind.
 138 pub fn test_main_static(tests: &[&TestDescAndFn]) {
 139     let args = env::args().collect::<Vec<_>>();
 140     let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
 141     test_main(&args, owned_tests, None)
 142 }
 143
 144 /// A variant optimized for invocation with a static test vector.
 145 /// This will panic (intentionally) when fed any dynamic tests.
 146 ///
 147 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
 148 /// tests.
 149 ///
 150 /// This is the entry point for the main function generated by `rustc --test`
 151 /// when panic=abort.
 152 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
 153     // If we're being run in SpawnedSecondary mode, run the test here. run_test
 154     // will then exit the process.
 155     if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
 156         env::remove_var(SECONDARY_TEST_INVOKER_VAR);
 157         let test = tests
 158             .iter()
 159             .filter(|test| test.desc.name.as_slice() == name)
 160             .map(make_owned_test)
 161             .next()
 162             .expect(&format!("couldn't find a test with the provided name '{}'", name));
 163         let TestDescAndFn { desc, testfn } = test;
 164         let testfn = match testfn {
 165             StaticTestFn(f) => f,
 166             _ => panic!("only static tests are supported"),
 167         };
 168         run_test_in_spawned_subprocess(desc, Box::new(testfn));
 169     }
 170
 171     let args = env::args().collect::<Vec<_>>();
 172     let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
 173     test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
 174 }
 175
 176 /// Clones static values for putting into a dynamic vector, which test_main()
 177 /// needs to hand out ownership of tests to parallel test runners.
 178 ///
 179 /// This will panic when fed any dynamic tests, because they cannot be cloned.
 180 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
 181     match test.testfn {
 182         StaticTestFn(f) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() },
 183         StaticBenchFn(f) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() },
 184         _ => panic!("non-static tests passed to test::test_main_static"),
 185     }
 186 }
 187
 188 /// Invoked when unit tests terminate. Should panic if the unit
 189 /// Tests is considered a failure. By default, invokes `report()`
 190 /// and checks for a `0` result.
 191 pub fn assert_test_result<T: Termination>(result: T) {
 192     let code = result.report();
 193     assert_eq!(
 194         code, 0,
 195         "the test returned a termination value with a non-zero status code ({}) \
 196          which indicates a failure",
 197         code
 198     );
 199 }
 200
 201 pub fn run_tests<F>(
 202     opts: &TestOpts,
 203     tests: Vec<TestDescAndFn>,
 204     mut notify_about_test_event: F,
 205 ) -> io::Result<()>
 206 where
 207     F: FnMut(TestEvent) -> io::Result<()>,
 208 {
 209     use std::collections::{self, HashMap};
 210     use std::hash::BuildHasherDefault;
 211     use std::sync::mpsc::RecvTimeoutError;
 212     // Use a deterministic hasher
 213     type TestMap =
 214         HashMap<TestDesc, Instant, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
 215
 216     let tests_len = tests.len();
 217
 218     let mut filtered_tests = filter_tests(opts, tests);
 219     if !opts.bench_benchmarks {
 220         filtered_tests = convert_benchmarks_to_tests(filtered_tests);
 221     }
 222
 223     let filtered_tests = {
 224         let mut filtered_tests = filtered_tests;
 225         for test in filtered_tests.iter_mut() {
 226             test.desc.name = test.desc.name.with_padding(test.testfn.padding());
 227         }
 228
 229         filtered_tests
 230     };
 231
 232     let filtered_out = tests_len - filtered_tests.len();
 233     let event = TestEvent::TeFilteredOut(filtered_out);
 234     notify_about_test_event(event)?;
 235
 236     let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
 237
 238     let event = TestEvent::TeFiltered(filtered_descs);
 239     notify_about_test_event(event)?;
 240
 241     let (filtered_tests, filtered_benchs): (Vec<_>, _) =
 242         filtered_tests.into_iter().partition(|e| match e.testfn {
 243             StaticTestFn(_) | DynTestFn(_) => true,
 244             _ => false,
 245         });
 246
 247     let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
 248
 249     let mut remaining = filtered_tests;
 250     remaining.reverse();
 251     let mut pending = 0;
 252
 253     let (tx, rx) = channel::<CompletedTest>();
 254     let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
 255         RunStrategy::SpawnPrimary
 256     } else {
 257         RunStrategy::InProcess
 258     };
 259
 260     let mut running_tests: TestMap = HashMap::default();
 261
 262     fn get_timed_out_tests(running_tests: &mut TestMap) -> Vec<TestDesc> {
 263         let now = Instant::now();
 264         let timed_out = running_tests
 265             .iter()
 266             .filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone()) } else { None })
 267             .collect();
 268         for test in &timed_out {
 269             running_tests.remove(test);
 270         }
 271         timed_out
 272     };
 273
 274     fn calc_timeout(running_tests: &TestMap) -> Option<Duration> {
 275         running_tests.values().min().map(|next_timeout| {
 276             let now = Instant::now();
 277             if *next_timeout >= now { *next_timeout - now } else { Duration::new(0, 0) }
 278         })
 279     };
 280
 281     if concurrency == 1 {
 282         while !remaining.is_empty() {
 283             let test = remaining.pop().unwrap();
 284             let event = TestEvent::TeWait(test.desc.clone());
 285             notify_about_test_event(event)?;
 286             run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No);
 287             let completed_test = rx.recv().unwrap();
 288
 289             let event = TestEvent::TeResult(completed_test);
 290             notify_about_test_event(event)?;
 291         }
 292     } else {
 293         while pending > 0 || !remaining.is_empty() {
 294             while pending < concurrency && !remaining.is_empty() {
 295                 let test = remaining.pop().unwrap();
 296                 let timeout = time::get_default_test_timeout();
 297                 running_tests.insert(test.desc.clone(), timeout);
 298
 299                 let event = TestEvent::TeWait(test.desc.clone());
 300                 notify_about_test_event(event)?; //here no pad
 301                 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::Yes);
 302                 pending += 1;
 303             }
 304
 305             let mut res;
 306             loop {
 307                 if let Some(timeout) = calc_timeout(&running_tests) {
 308                     res = rx.recv_timeout(timeout);
 309                     for test in get_timed_out_tests(&mut running_tests) {
 310                         let event = TestEvent::TeTimeout(test);
 311                         notify_about_test_event(event)?;
 312                     }
 313
 314                     match res {
 315                         Err(RecvTimeoutError::Timeout) => {
 316                             // Result is not yet ready, continue waiting.
 317                         }
 318                         _ => {
 319                             // We've got a result, stop the loop.
 320                             break;
 321                         }
 322                     }
 323                 } else {
 324                     res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
 325                     break;
 326                 }
 327             }
 328
 329             let completed_test = res.unwrap();
 330             running_tests.remove(&completed_test.desc);
 331
 332             let event = TestEvent::TeResult(completed_test);
 333             notify_about_test_event(event)?;
 334             pending -= 1;
 335         }
 336     }
 337
 338     if opts.bench_benchmarks {
 339         // All benchmarks run at the end, in serial.
 340         for b in filtered_benchs {
 341             let event = TestEvent::TeWait(b.desc.clone());
 342             notify_about_test_event(event)?;
 343             run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
 344             let completed_test = rx.recv().unwrap();
 345
 346             let event = TestEvent::TeResult(completed_test);
 347             notify_about_test_event(event)?;
 348         }
 349     }
 350     Ok(())
 351 }
 352
 353 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
 354     let mut filtered = tests;
 355     let matches_filter = |test: &TestDescAndFn, filter: &str| {
 356         let test_name = test.desc.name.as_slice();
 357
 358         match opts.filter_exact {
 359             true => test_name == filter,
 360             false => test_name.contains(filter),
 361         }
 362     };
 363
 364     // Remove tests that don't match the test filter
 365     if let Some(ref filter) = opts.filter {
 366         filtered.retain(|test| matches_filter(test, filter));
 367     }
 368
 369     // Skip tests that match any of the skip filters
 370     filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
 371
 372     // Excludes #[should_panic] tests
 373     if opts.exclude_should_panic {
 374         filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
 375     }
 376
 377     // maybe unignore tests
 378     match opts.run_ignored {
 379         RunIgnored::Yes => {
 380             filtered.iter_mut().for_each(|test| test.desc.ignore = false);
 381         }
 382         RunIgnored::Only => {
 383             filtered.retain(|test| test.desc.ignore);
 384             filtered.iter_mut().for_each(|test| test.desc.ignore = false);
 385         }
 386         RunIgnored::No => {}
 387     }
 388
 389     // Sort the tests alphabetically
 390     filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
 391
 392     filtered
 393 }
 394
 395 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
 396     // convert benchmarks to tests, if we're not benchmarking them
 397     tests
 398         .into_iter()
 399         .map(|x| {
 400             let testfn = match x.testfn {
 401                 DynBenchFn(bench) => DynTestFn(Box::new(move || {
 402                     bench::run_once(|b| __rust_begin_short_backtrace(|| bench.run(b)))
 403                 })),
 404                 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
 405                     bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
 406                 })),
 407                 f => f,
 408             };
 409             TestDescAndFn { desc: x.desc, testfn }
 410         })
 411         .collect()
 412 }
 413
 414 pub fn run_test(
 415     opts: &TestOpts,
 416     force_ignore: bool,
 417     test: TestDescAndFn,
 418     strategy: RunStrategy,
 419     monitor_ch: Sender<CompletedTest>,
 420     concurrency: Concurrent,
 421 ) {
 422     let TestDescAndFn { desc, testfn } = test;
 423
 424     // Emscripten can catch panics but other wasm targets cannot
 425     let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
 426         && cfg!(target_arch = "wasm32")
 427         && !cfg!(target_os = "emscripten");
 428
 429     if force_ignore || desc.ignore || ignore_because_no_process_support {
 430         let message = CompletedTest::new(desc, TrIgnored, None, Vec::new());
 431         monitor_ch.send(message).unwrap();
 432         return;
 433     }
 434
 435     struct TestRunOpts {
 436         pub strategy: RunStrategy,
 437         pub nocapture: bool,
 438         pub concurrency: Concurrent,
 439         pub time: Option<time::TestTimeOptions>,
 440     }
 441
 442     fn run_test_inner(
 443         desc: TestDesc,
 444         monitor_ch: Sender<CompletedTest>,
 445         testfn: Box<dyn FnOnce() + Send>,
 446         opts: TestRunOpts,
 447     ) {
 448         let concurrency = opts.concurrency;
 449         let name = desc.name.clone();
 450
 451         let runtest = move || match opts.strategy {
 452             RunStrategy::InProcess => run_test_in_process(
 453                 desc,
 454                 opts.nocapture,
 455                 opts.time.is_some(),
 456                 testfn,
 457                 monitor_ch,
 458                 opts.time,
 459             ),
 460             RunStrategy::SpawnPrimary => {
 461                 spawn_test_subprocess(desc, opts.time.is_some(), monitor_ch, opts.time)
 462             }
 463         };
 464
 465         // If the platform is single-threaded we're just going to run
 466         // the test synchronously, regardless of the concurrency
 467         // level.
 468         let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
 469         if concurrency == Concurrent::Yes && supports_threads {
 470             let cfg = thread::Builder::new().name(name.as_slice().to_owned());
 471             cfg.spawn(runtest).unwrap();
 472         } else {
 473             runtest();
 474         }
 475     }
 476
 477     let test_run_opts =
 478         TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options };
 479
 480     match testfn {
 481         DynBenchFn(bencher) => {
 482             // Benchmarks aren't expected to panic, so we run them all in-process.
 483             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
 484                 bencher.run(harness)
 485             });
 486         }
 487         StaticBenchFn(benchfn) => {
 488             // Benchmarks aren't expected to panic, so we run them all in-process.
 489             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, benchfn);
 490         }
 491         DynTestFn(f) => {
 492             match strategy {
 493                 RunStrategy::InProcess => (),
 494                 _ => panic!("Cannot run dynamic test fn out-of-process"),
 495             };
 496             run_test_inner(
 497                 desc,
 498                 monitor_ch,
 499                 Box::new(move || __rust_begin_short_backtrace(f)),
 500                 test_run_opts,
 501             );
 502         }
 503         StaticTestFn(f) => run_test_inner(
 504             desc,
 505             monitor_ch,
 506             Box::new(move || __rust_begin_short_backtrace(f)),
 507             test_run_opts,
 508         ),
 509     }
 510 }
 511
 512 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
 513 #[inline(never)]
 514 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
 515     f()
 516 }
 517
 518 fn run_test_in_process(
 519     desc: TestDesc,
 520     nocapture: bool,
 521     report_time: bool,
 522     testfn: Box<dyn FnOnce() + Send>,
 523     monitor_ch: Sender<CompletedTest>,
 524     time_opts: Option<time::TestTimeOptions>,
 525 ) {
 526     // Buffer for capturing standard I/O
 527     let data = Arc::new(Mutex::new(Vec::new()));
 528
 529     let oldio = if !nocapture {
 530         Some((
 531             io::set_print(Some(Sink::new_boxed(&data))),
 532             io::set_panic(Some(Sink::new_boxed(&data))),
 533         ))
 534     } else {
 535         None
 536     };
 537
 538     let start = report_time.then(Instant::now);
 539     let result = catch_unwind(AssertUnwindSafe(testfn));
 540     let exec_time = start.map(|start| {
 541         let duration = start.elapsed();
 542         TestExecTime(duration)
 543     });
 544
 545     if let Some((printio, panicio)) = oldio {
 546         io::set_print(printio);
 547         io::set_panic(panicio);
 548     }
 549
 550     let test_result = match result {
 551         Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
 552         Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
 553     };
 554     let stdout = data.lock().unwrap().to_vec();
 555     let message = CompletedTest::new(desc, test_result, exec_time, stdout);
 556     monitor_ch.send(message).unwrap();
 557 }
 558
 559 fn spawn_test_subprocess(
 560     desc: TestDesc,
 561     report_time: bool,
 562     monitor_ch: Sender<CompletedTest>,
 563     time_opts: Option<time::TestTimeOptions>,
 564 ) {
 565     let (result, test_output, exec_time) = (|| {
 566         let args = env::args().collect::<Vec<_>>();
 567         let current_exe = &args[0];
 568
 569         let start = report_time.then(Instant::now);
 570         let output = match Command::new(current_exe)
 571             .env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice())
 572             .output()
 573         {
 574             Ok(out) => out,
 575             Err(e) => {
 576                 let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
 577                 return (TrFailed, err.into_bytes(), None);
 578             }
 579         };
 580         let exec_time = start.map(|start| {
 581             let duration = start.elapsed();
 582             TestExecTime(duration)
 583         });
 584
 585         let std::process::Output { stdout, stderr, status } = output;
 586         let mut test_output = stdout;
 587         formatters::write_stderr_delimiter(&mut test_output, &desc.name);
 588         test_output.extend_from_slice(&stderr);
 589
 590         let result = match (|| -> Result<TestResult, String> {
 591             let exit_code = get_exit_code(status)?;
 592             Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
 593         })() {
 594             Ok(r) => r,
 595             Err(e) => {
 596                 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
 597                 TrFailed
 598             }
 599         };
 600
 601         (result, test_output, exec_time)
 602     })();
 603
 604     let message = CompletedTest::new(desc, result, exec_time, test_output);
 605     monitor_ch.send(message).unwrap();
 606 }
 607
 608 fn run_test_in_spawned_subprocess(desc: TestDesc, testfn: Box<dyn FnOnce() + Send>) -> ! {
 609     let builtin_panic_hook = panic::take_hook();
 610     let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
 611         let test_result = match panic_info {
 612             Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
 613             None => calc_result(&desc, Ok(()), &None, &None),
 614         };
 615
 616         // We don't support serializing TrFailedMsg, so just
 617         // print the message out to stderr.
 618         if let TrFailedMsg(msg) = &test_result {
 619             eprintln!("{}", msg);
 620         }
 621
 622         if let Some(info) = panic_info {
 623             builtin_panic_hook(info);
 624         }
 625
 626         if let TrOk = test_result {
 627             process::exit(test_result::TR_OK);
 628         } else {
 629             process::exit(test_result::TR_FAILED);
 630         }
 631     });
 632     let record_result2 = record_result.clone();
 633     panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
 634     testfn();
 635     record_result(None);
 636     unreachable!("panic=abort callback should have exited the process")
 637 }