1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
16 // N.B., this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
24 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
25 #![feature(rustc_private)]
27 #![feature(bool_to_option)]
28 #![feature(set_stdio)]
29 #![feature(panic_unwind)]
30 #![feature(staged_api)]
31 #![feature(termination_trait_lib)]
35 pub use self::bench::{black_box, Bencher};
36 pub use self::console::run_tests_console;
37 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
38 pub use self::types::TestName::*;
39 pub use self::types::*;
40 pub use self::ColorConfig::*;
41 pub use cli::TestOpts;
43 // Module to be used by rustc to compile tests in libtest
48 cli::{parse_opts, TestOpts},
50 helpers::metrics::{Metric, MetricMap},
51 options::{Options, RunIgnored, RunStrategy, ShouldPanic},
52 run_test, test_main, test_main_static,
53 test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
54 time::{TestExecTime, TestTimeOptions},
56 DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc,
57 TestDescAndFn, TestName, TestType,
65 panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
67 process::{Command, Termination},
68 sync::mpsc::{channel, Sender},
71 time::{Duration, Instant},
89 use event::{CompletedTest, TestEvent};
90 use helpers::concurrency::get_concurrency;
91 use helpers::exit_code::get_exit_code;
92 use helpers::sink::Sink;
93 use options::{Concurrent, RunStrategy};
95 use time::TestExecTime;
97 // Process exit code to be used to indicate test failures.
98 const ERROR_EXIT_CODE: i32 = 101;
100 const SECONDARY_TEST_INVOKER_VAR: &'static str = "__RUST_TEST_INVOKE";
102 // The default console test runner. It accepts the command line
103 // arguments and a vector of test_descs.
104 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
105 let mut opts = match cli::parse_opts(args) {
108 eprintln!("error: {}", msg);
109 process::exit(ERROR_EXIT_CODE);
113 if let Some(options) = options {
114 opts.options = options;
117 if let Err(e) = console::list_tests_console(&opts, tests) {
118 eprintln!("error: io error when listing tests: {:?}", e);
119 process::exit(ERROR_EXIT_CODE);
122 match console::run_tests_console(&opts, tests) {
124 Ok(false) => process::exit(ERROR_EXIT_CODE),
126 eprintln!("error: io error when listing tests: {:?}", e);
127 process::exit(ERROR_EXIT_CODE);
133 /// A variant optimized for invocation with a static test vector.
134 /// This will panic (intentionally) when fed any dynamic tests.
136 /// This is the entry point for the main function generated by `rustc --test`
137 /// when panic=unwind.
138 pub fn test_main_static(tests: &[&TestDescAndFn]) {
139 let args = env::args().collect::<Vec<_>>();
140 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
141 test_main(&args, owned_tests, None)
144 /// A variant optimized for invocation with a static test vector.
145 /// This will panic (intentionally) when fed any dynamic tests.
147 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
150 /// This is the entry point for the main function generated by `rustc --test`
151 /// when panic=abort.
152 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
153 // If we're being run in SpawnedSecondary mode, run the test here. run_test
154 // will then exit the process.
155 if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
156 env::remove_var(SECONDARY_TEST_INVOKER_VAR);
159 .filter(|test| test.desc.name.as_slice() == name)
160 .map(make_owned_test)
162 .expect(&format!("couldn't find a test with the provided name '{}'", name));
163 let TestDescAndFn { desc, testfn } = test;
164 let testfn = match testfn {
165 StaticTestFn(f) => f,
166 _ => panic!("only static tests are supported"),
168 run_test_in_spawned_subprocess(desc, Box::new(testfn));
171 let args = env::args().collect::<Vec<_>>();
172 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
173 test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
176 /// Clones static values for putting into a dynamic vector, which test_main()
177 /// needs to hand out ownership of tests to parallel test runners.
179 /// This will panic when fed any dynamic tests, because they cannot be cloned.
180 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
182 StaticTestFn(f) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() },
183 StaticBenchFn(f) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() },
184 _ => panic!("non-static tests passed to test::test_main_static"),
188 /// Invoked when unit tests terminate. Should panic if the unit
189 /// Tests is considered a failure. By default, invokes `report()`
190 /// and checks for a `0` result.
191 pub fn assert_test_result<T: Termination>(result: T) {
192 let code = result.report();
195 "the test returned a termination value with a non-zero status code ({}) \
196 which indicates a failure",
203 tests: Vec<TestDescAndFn>,
204 mut notify_about_test_event: F,
207 F: FnMut(TestEvent) -> io::Result<()>,
209 use std::collections::{self, HashMap};
210 use std::hash::BuildHasherDefault;
211 use std::sync::mpsc::RecvTimeoutError;
212 // Use a deterministic hasher
214 HashMap<TestDesc, Instant, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
216 let tests_len = tests.len();
218 let mut filtered_tests = filter_tests(opts, tests);
219 if !opts.bench_benchmarks {
220 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
223 let filtered_tests = {
224 let mut filtered_tests = filtered_tests;
225 for test in filtered_tests.iter_mut() {
226 test.desc.name = test.desc.name.with_padding(test.testfn.padding());
232 let filtered_out = tests_len - filtered_tests.len();
233 let event = TestEvent::TeFilteredOut(filtered_out);
234 notify_about_test_event(event)?;
236 let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
238 let event = TestEvent::TeFiltered(filtered_descs);
239 notify_about_test_event(event)?;
241 let (filtered_tests, filtered_benchs): (Vec<_>, _) =
242 filtered_tests.into_iter().partition(|e| match e.testfn {
243 StaticTestFn(_) | DynTestFn(_) => true,
247 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
249 let mut remaining = filtered_tests;
253 let (tx, rx) = channel::<CompletedTest>();
254 let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
255 RunStrategy::SpawnPrimary
257 RunStrategy::InProcess
260 let mut running_tests: TestMap = HashMap::default();
262 fn get_timed_out_tests(running_tests: &mut TestMap) -> Vec<TestDesc> {
263 let now = Instant::now();
264 let timed_out = running_tests
266 .filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone()) } else { None })
268 for test in &timed_out {
269 running_tests.remove(test);
274 fn calc_timeout(running_tests: &TestMap) -> Option<Duration> {
275 running_tests.values().min().map(|next_timeout| {
276 let now = Instant::now();
277 if *next_timeout >= now { *next_timeout - now } else { Duration::new(0, 0) }
281 if concurrency == 1 {
282 while !remaining.is_empty() {
283 let test = remaining.pop().unwrap();
284 let event = TestEvent::TeWait(test.desc.clone());
285 notify_about_test_event(event)?;
286 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No);
287 let completed_test = rx.recv().unwrap();
289 let event = TestEvent::TeResult(completed_test);
290 notify_about_test_event(event)?;
293 while pending > 0 || !remaining.is_empty() {
294 while pending < concurrency && !remaining.is_empty() {
295 let test = remaining.pop().unwrap();
296 let timeout = time::get_default_test_timeout();
297 running_tests.insert(test.desc.clone(), timeout);
299 let event = TestEvent::TeWait(test.desc.clone());
300 notify_about_test_event(event)?; //here no pad
301 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::Yes);
307 if let Some(timeout) = calc_timeout(&running_tests) {
308 res = rx.recv_timeout(timeout);
309 for test in get_timed_out_tests(&mut running_tests) {
310 let event = TestEvent::TeTimeout(test);
311 notify_about_test_event(event)?;
315 Err(RecvTimeoutError::Timeout) => {
316 // Result is not yet ready, continue waiting.
319 // We've got a result, stop the loop.
324 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
329 let completed_test = res.unwrap();
330 running_tests.remove(&completed_test.desc);
332 let event = TestEvent::TeResult(completed_test);
333 notify_about_test_event(event)?;
338 if opts.bench_benchmarks {
339 // All benchmarks run at the end, in serial.
340 for b in filtered_benchs {
341 let event = TestEvent::TeWait(b.desc.clone());
342 notify_about_test_event(event)?;
343 run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
344 let completed_test = rx.recv().unwrap();
346 let event = TestEvent::TeResult(completed_test);
347 notify_about_test_event(event)?;
353 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
354 let mut filtered = tests;
355 let matches_filter = |test: &TestDescAndFn, filter: &str| {
356 let test_name = test.desc.name.as_slice();
358 match opts.filter_exact {
359 true => test_name == filter,
360 false => test_name.contains(filter),
364 // Remove tests that don't match the test filter
365 if let Some(ref filter) = opts.filter {
366 filtered.retain(|test| matches_filter(test, filter));
369 // Skip tests that match any of the skip filters
370 filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
372 // Excludes #[should_panic] tests
373 if opts.exclude_should_panic {
374 filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
377 // maybe unignore tests
378 match opts.run_ignored {
380 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
382 RunIgnored::Only => {
383 filtered.retain(|test| test.desc.ignore);
384 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
389 // Sort the tests alphabetically
390 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
395 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
396 // convert benchmarks to tests, if we're not benchmarking them
400 let testfn = match x.testfn {
401 DynBenchFn(bench) => DynTestFn(Box::new(move || {
402 bench::run_once(|b| __rust_begin_short_backtrace(|| bench.run(b)))
404 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
405 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
409 TestDescAndFn { desc: x.desc, testfn }
418 strategy: RunStrategy,
419 monitor_ch: Sender<CompletedTest>,
420 concurrency: Concurrent,
422 let TestDescAndFn { desc, testfn } = test;
424 // Emscripten can catch panics but other wasm targets cannot
425 let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
426 && cfg!(target_arch = "wasm32")
427 && !cfg!(target_os = "emscripten");
429 if force_ignore || desc.ignore || ignore_because_no_process_support {
430 let message = CompletedTest::new(desc, TrIgnored, None, Vec::new());
431 monitor_ch.send(message).unwrap();
436 pub strategy: RunStrategy,
438 pub concurrency: Concurrent,
439 pub time: Option<time::TestTimeOptions>,
444 monitor_ch: Sender<CompletedTest>,
445 testfn: Box<dyn FnOnce() + Send>,
448 let concurrency = opts.concurrency;
449 let name = desc.name.clone();
451 let runtest = move || match opts.strategy {
452 RunStrategy::InProcess => run_test_in_process(
460 RunStrategy::SpawnPrimary => {
461 spawn_test_subprocess(desc, opts.time.is_some(), monitor_ch, opts.time)
465 // If the platform is single-threaded we're just going to run
466 // the test synchronously, regardless of the concurrency
468 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
469 if concurrency == Concurrent::Yes && supports_threads {
470 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
471 cfg.spawn(runtest).unwrap();
478 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options };
481 DynBenchFn(bencher) => {
482 // Benchmarks aren't expected to panic, so we run them all in-process.
483 crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
487 StaticBenchFn(benchfn) => {
488 // Benchmarks aren't expected to panic, so we run them all in-process.
489 crate::bench::benchmark(desc, monitor_ch, opts.nocapture, benchfn);
493 RunStrategy::InProcess => (),
494 _ => panic!("Cannot run dynamic test fn out-of-process"),
499 Box::new(move || __rust_begin_short_backtrace(f)),
503 StaticTestFn(f) => run_test_inner(
506 Box::new(move || __rust_begin_short_backtrace(f)),
512 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
514 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
518 fn run_test_in_process(
522 testfn: Box<dyn FnOnce() + Send>,
523 monitor_ch: Sender<CompletedTest>,
524 time_opts: Option<time::TestTimeOptions>,
526 // Buffer for capturing standard I/O
527 let data = Arc::new(Mutex::new(Vec::new()));
529 let oldio = if !nocapture {
531 io::set_print(Some(Sink::new_boxed(&data))),
532 io::set_panic(Some(Sink::new_boxed(&data))),
538 let start = report_time.then(Instant::now);
539 let result = catch_unwind(AssertUnwindSafe(testfn));
540 let exec_time = start.map(|start| {
541 let duration = start.elapsed();
542 TestExecTime(duration)
545 if let Some((printio, panicio)) = oldio {
546 io::set_print(printio);
547 io::set_panic(panicio);
550 let test_result = match result {
551 Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
552 Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
554 let stdout = data.lock().unwrap().to_vec();
555 let message = CompletedTest::new(desc, test_result, exec_time, stdout);
556 monitor_ch.send(message).unwrap();
559 fn spawn_test_subprocess(
562 monitor_ch: Sender<CompletedTest>,
563 time_opts: Option<time::TestTimeOptions>,
565 let (result, test_output, exec_time) = (|| {
566 let args = env::args().collect::<Vec<_>>();
567 let current_exe = &args[0];
569 let start = report_time.then(Instant::now);
570 let output = match Command::new(current_exe)
571 .env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice())
576 let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
577 return (TrFailed, err.into_bytes(), None);
580 let exec_time = start.map(|start| {
581 let duration = start.elapsed();
582 TestExecTime(duration)
585 let std::process::Output { stdout, stderr, status } = output;
586 let mut test_output = stdout;
587 formatters::write_stderr_delimiter(&mut test_output, &desc.name);
588 test_output.extend_from_slice(&stderr);
590 let result = match (|| -> Result<TestResult, String> {
591 let exit_code = get_exit_code(status)?;
592 Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
596 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
601 (result, test_output, exec_time)
604 let message = CompletedTest::new(desc, result, exec_time, test_output);
605 monitor_ch.send(message).unwrap();
608 fn run_test_in_spawned_subprocess(desc: TestDesc, testfn: Box<dyn FnOnce() + Send>) -> ! {
609 let builtin_panic_hook = panic::take_hook();
610 let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
611 let test_result = match panic_info {
612 Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
613 None => calc_result(&desc, Ok(()), &None, &None),
616 // We don't support serializing TrFailedMsg, so just
617 // print the message out to stderr.
618 if let TrFailedMsg(msg) = &test_result {
619 eprintln!("{}", msg);
622 if let Some(info) = panic_info {
623 builtin_panic_hook(info);
626 if let TrOk = test_result {
627 process::exit(test_result::TR_OK);
629 process::exit(test_result::TR_FAILED);
632 let record_result2 = record_result.clone();
633 panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
636 unreachable!("panic=abort callback should have exited the process")