1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
16 // N.B., this is also specified in this crate's Cargo.toml, but librustc_ast contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
23 #![cfg_attr(unix, feature(libc))]
24 #![feature(rustc_private)]
26 #![feature(available_concurrency)]
27 #![feature(internal_output_capture)]
28 #![feature(option_unwrap_none)]
29 #![feature(panic_unwind)]
30 #![feature(staged_api)]
31 #![feature(termination_trait_lib)]
33 #![feature(total_cmp)]
34 #![feature(str_split_once)]
37 pub use self::bench::{black_box, Bencher};
38 pub use self::console::run_tests_console;
39 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
40 pub use self::types::TestName::*;
41 pub use self::types::*;
42 pub use self::ColorConfig::*;
43 pub use cli::TestOpts;
45 // Module to be used by rustc to compile tests in libtest
50 cli::{parse_opts, TestOpts},
52 helpers::metrics::{Metric, MetricMap},
53 options::{Options, RunIgnored, RunStrategy, ShouldPanic},
54 run_test, test_main, test_main_static,
55 test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
56 time::{TestExecTime, TestTimeOptions},
58 DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc,
59 TestDescAndFn, TestName, TestType,
65 collections::VecDeque,
68 panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
69 process::{self, Command, Termination},
70 sync::mpsc::{channel, Sender},
73 time::{Duration, Instant},
91 use event::{CompletedTest, TestEvent};
92 use helpers::concurrency::get_concurrency;
93 use helpers::exit_code::get_exit_code;
94 use options::{Concurrent, RunStrategy};
96 use time::TestExecTime;
98 // Process exit code to be used to indicate test failures.
99 const ERROR_EXIT_CODE: i32 = 101;
101 const SECONDARY_TEST_INVOKER_VAR: &str = "__RUST_TEST_INVOKE";
103 // The default console test runner. It accepts the command line
104 // arguments and a vector of test_descs.
105 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
106 let mut opts = match cli::parse_opts(args) {
109 eprintln!("error: {}", msg);
110 process::exit(ERROR_EXIT_CODE);
114 if let Some(options) = options {
115 opts.options = options;
118 if let Err(e) = console::list_tests_console(&opts, tests) {
119 eprintln!("error: io error when listing tests: {:?}", e);
120 process::exit(ERROR_EXIT_CODE);
123 match console::run_tests_console(&opts, tests) {
125 Ok(false) => process::exit(ERROR_EXIT_CODE),
127 eprintln!("error: io error when listing tests: {:?}", e);
128 process::exit(ERROR_EXIT_CODE);
134 /// A variant optimized for invocation with a static test vector.
135 /// This will panic (intentionally) when fed any dynamic tests.
137 /// This is the entry point for the main function generated by `rustc --test`
138 /// when panic=unwind.
139 pub fn test_main_static(tests: &[&TestDescAndFn]) {
140 let args = env::args().collect::<Vec<_>>();
141 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
142 test_main(&args, owned_tests, None)
145 /// A variant optimized for invocation with a static test vector.
146 /// This will panic (intentionally) when fed any dynamic tests.
148 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
151 /// This is the entry point for the main function generated by `rustc --test`
152 /// when panic=abort.
153 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
154 // If we're being run in SpawnedSecondary mode, run the test here. run_test
155 // will then exit the process.
156 if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
157 env::remove_var(SECONDARY_TEST_INVOKER_VAR);
160 .filter(|test| test.desc.name.as_slice() == name)
161 .map(make_owned_test)
163 .unwrap_or_else(|| panic!("couldn't find a test with the provided name '{}'", name));
164 let TestDescAndFn { desc, testfn } = test;
165 let testfn = match testfn {
166 StaticTestFn(f) => f,
167 _ => panic!("only static tests are supported"),
169 run_test_in_spawned_subprocess(desc, Box::new(testfn));
172 let args = env::args().collect::<Vec<_>>();
173 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
174 test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
177 /// Clones static values for putting into a dynamic vector, which test_main()
178 /// needs to hand out ownership of tests to parallel test runners.
180 /// This will panic when fed any dynamic tests, because they cannot be cloned.
181 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
183 StaticTestFn(f) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() },
184 StaticBenchFn(f) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() },
185 _ => panic!("non-static tests passed to test::test_main_static"),
189 /// Invoked when unit tests terminate. Should panic if the unit
190 /// Tests is considered a failure. By default, invokes `report()`
191 /// and checks for a `0` result.
192 pub fn assert_test_result<T: Termination>(result: T) {
193 let code = result.report();
196 "the test returned a termination value with a non-zero status code ({}) \
197 which indicates a failure",
204 tests: Vec<TestDescAndFn>,
205 mut notify_about_test_event: F,
208 F: FnMut(TestEvent) -> io::Result<()>,
210 use std::collections::{self, HashMap};
211 use std::hash::BuildHasherDefault;
212 use std::sync::mpsc::RecvTimeoutError;
215 join_handle: Option<thread::JoinHandle<()>>,
218 // Use a deterministic hasher
220 HashMap<TestDesc, RunningTest, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
222 struct TimeoutEntry {
227 let tests_len = tests.len();
229 let mut filtered_tests = filter_tests(opts, tests);
230 if !opts.bench_benchmarks {
231 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
234 let filtered_tests = {
235 let mut filtered_tests = filtered_tests;
236 for test in filtered_tests.iter_mut() {
237 test.desc.name = test.desc.name.with_padding(test.testfn.padding());
243 let filtered_out = tests_len - filtered_tests.len();
244 let event = TestEvent::TeFilteredOut(filtered_out);
245 notify_about_test_event(event)?;
247 let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
249 let event = TestEvent::TeFiltered(filtered_descs);
250 notify_about_test_event(event)?;
252 let (filtered_tests, filtered_benchs): (Vec<_>, _) = filtered_tests
254 .partition(|e| matches!(e.testfn, StaticTestFn(_) | DynTestFn(_)));
256 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
258 let mut remaining = filtered_tests;
262 let (tx, rx) = channel::<CompletedTest>();
263 let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
264 RunStrategy::SpawnPrimary
266 RunStrategy::InProcess
269 let mut running_tests: TestMap = HashMap::default();
270 let mut timeout_queue: VecDeque<TimeoutEntry> = VecDeque::new();
272 fn get_timed_out_tests(
273 running_tests: &TestMap,
274 timeout_queue: &mut VecDeque<TimeoutEntry>,
276 let now = Instant::now();
277 let mut timed_out = Vec::new();
278 while let Some(timeout_entry) = timeout_queue.front() {
279 if now < timeout_entry.timeout {
282 let timeout_entry = timeout_queue.pop_front().unwrap();
283 if running_tests.contains_key(&timeout_entry.desc) {
284 timed_out.push(timeout_entry.desc);
290 fn calc_timeout(timeout_queue: &VecDeque<TimeoutEntry>) -> Option<Duration> {
291 timeout_queue.front().map(|&TimeoutEntry { timeout: next_timeout, .. }| {
292 let now = Instant::now();
293 if next_timeout >= now { next_timeout - now } else { Duration::new(0, 0) }
297 if concurrency == 1 {
298 while !remaining.is_empty() {
299 let test = remaining.pop().unwrap();
300 let event = TestEvent::TeWait(test.desc.clone());
301 notify_about_test_event(event)?;
302 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No)
304 let completed_test = rx.recv().unwrap();
306 let event = TestEvent::TeResult(completed_test);
307 notify_about_test_event(event)?;
310 while pending > 0 || !remaining.is_empty() {
311 while pending < concurrency && !remaining.is_empty() {
312 let test = remaining.pop().unwrap();
313 let timeout = time::get_default_test_timeout();
314 let desc = test.desc.clone();
316 let event = TestEvent::TeWait(desc.clone());
317 notify_about_test_event(event)?; //here no pad
318 let join_handle = run_test(
326 running_tests.insert(desc.clone(), RunningTest { join_handle });
327 timeout_queue.push_back(TimeoutEntry { desc, timeout });
333 if let Some(timeout) = calc_timeout(&timeout_queue) {
334 res = rx.recv_timeout(timeout);
335 for test in get_timed_out_tests(&running_tests, &mut timeout_queue) {
336 let event = TestEvent::TeTimeout(test);
337 notify_about_test_event(event)?;
341 Err(RecvTimeoutError::Timeout) => {
342 // Result is not yet ready, continue waiting.
345 // We've got a result, stop the loop.
350 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
355 let mut completed_test = res.unwrap();
356 if let Some(running_test) = running_tests.remove(&completed_test.desc) {
357 if let Some(join_handle) = running_test.join_handle {
358 if let Err(_) = join_handle.join() {
359 if let TrOk = completed_test.result {
360 completed_test.result =
361 TrFailedMsg("panicked after reporting success".to_string());
367 let event = TestEvent::TeResult(completed_test);
368 notify_about_test_event(event)?;
373 if opts.bench_benchmarks {
374 // All benchmarks run at the end, in serial.
375 for b in filtered_benchs {
376 let event = TestEvent::TeWait(b.desc.clone());
377 notify_about_test_event(event)?;
378 run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
379 let completed_test = rx.recv().unwrap();
381 let event = TestEvent::TeResult(completed_test);
382 notify_about_test_event(event)?;
388 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
389 let mut filtered = tests;
390 let matches_filter = |test: &TestDescAndFn, filter: &str| {
391 let test_name = test.desc.name.as_slice();
393 match opts.filter_exact {
394 true => test_name == filter,
395 false => test_name.contains(filter),
399 // Remove tests that don't match the test filter
400 if !opts.filters.is_empty() {
401 filtered.retain(|test| opts.filters.iter().any(|filter| matches_filter(test, filter)));
404 // Skip tests that match any of the skip filters
405 filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
407 // Excludes #[should_panic] tests
408 if opts.exclude_should_panic {
409 filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
412 // maybe unignore tests
413 match opts.run_ignored {
415 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
417 RunIgnored::Only => {
418 filtered.retain(|test| test.desc.ignore);
419 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
424 // Sort the tests alphabetically
425 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
430 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
431 // convert benchmarks to tests, if we're not benchmarking them
435 let testfn = match x.testfn {
436 DynBenchFn(bench) => DynTestFn(Box::new(move || {
437 bench::run_once(|b| __rust_begin_short_backtrace(|| bench.run(b)))
439 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
440 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
444 TestDescAndFn { desc: x.desc, testfn }
453 strategy: RunStrategy,
454 monitor_ch: Sender<CompletedTest>,
455 concurrency: Concurrent,
456 ) -> Option<thread::JoinHandle<()>> {
457 let TestDescAndFn { desc, testfn } = test;
459 // Emscripten can catch panics but other wasm targets cannot
460 let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
461 && cfg!(target_arch = "wasm32")
462 && !cfg!(target_os = "emscripten");
464 if force_ignore || desc.ignore || ignore_because_no_process_support {
465 let message = CompletedTest::new(desc, TrIgnored, None, Vec::new());
466 monitor_ch.send(message).unwrap();
471 pub strategy: RunStrategy,
473 pub concurrency: Concurrent,
474 pub time: Option<time::TestTimeOptions>,
479 monitor_ch: Sender<CompletedTest>,
480 testfn: Box<dyn FnOnce() + Send>,
482 ) -> Option<thread::JoinHandle<()>> {
483 let concurrency = opts.concurrency;
484 let name = desc.name.clone();
486 let runtest = move || match opts.strategy {
487 RunStrategy::InProcess => run_test_in_process(
495 RunStrategy::SpawnPrimary => spawn_test_subprocess(
504 // If the platform is single-threaded we're just going to run
505 // the test synchronously, regardless of the concurrency
507 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
508 if concurrency == Concurrent::Yes && supports_threads {
509 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
510 let mut runtest = Arc::new(Mutex::new(Some(runtest)));
511 let runtest2 = runtest.clone();
512 match cfg.spawn(move || runtest2.lock().unwrap().take().unwrap()()) {
513 Ok(handle) => Some(handle),
514 Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
515 // `ErrorKind::WouldBlock` means hitting the thread limit on some
516 // platforms, so run the test synchronously here instead.
517 Arc::get_mut(&mut runtest).unwrap().get_mut().unwrap().take().unwrap()();
520 Err(e) => panic!("failed to spawn thread to run test: {}", e),
529 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options };
532 DynBenchFn(bencher) => {
533 // Benchmarks aren't expected to panic, so we run them all in-process.
534 crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
539 StaticBenchFn(benchfn) => {
540 // Benchmarks aren't expected to panic, so we run them all in-process.
541 crate::bench::benchmark(desc, monitor_ch, opts.nocapture, benchfn);
546 RunStrategy::InProcess => (),
547 _ => panic!("Cannot run dynamic test fn out-of-process"),
552 Box::new(move || __rust_begin_short_backtrace(f)),
556 StaticTestFn(f) => run_test_inner(
559 Box::new(move || __rust_begin_short_backtrace(f)),
565 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
567 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
570 // prevent this frame from being tail-call optimised away
574 fn run_test_in_process(
578 testfn: Box<dyn FnOnce() + Send>,
579 monitor_ch: Sender<CompletedTest>,
580 time_opts: Option<time::TestTimeOptions>,
582 // Buffer for capturing standard I/O
583 let data = Arc::new(Mutex::new(Vec::new()));
586 io::set_output_capture(Some(data.clone()));
589 let start = report_time.then(Instant::now);
590 let result = catch_unwind(AssertUnwindSafe(testfn));
591 let exec_time = start.map(|start| {
592 let duration = start.elapsed();
593 TestExecTime(duration)
596 io::set_output_capture(None);
598 let test_result = match result {
599 Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
600 Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
602 let stdout = data.lock().unwrap_or_else(|e| e.into_inner()).to_vec();
603 let message = CompletedTest::new(desc, test_result, exec_time, stdout);
604 monitor_ch.send(message).unwrap();
607 fn spawn_test_subprocess(
611 monitor_ch: Sender<CompletedTest>,
612 time_opts: Option<time::TestTimeOptions>,
614 let (result, test_output, exec_time) = (|| {
615 let args = env::args().collect::<Vec<_>>();
616 let current_exe = &args[0];
618 let mut command = Command::new(current_exe);
619 command.env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice());
621 command.stdout(process::Stdio::inherit());
622 command.stderr(process::Stdio::inherit());
625 let start = report_time.then(Instant::now);
626 let output = match command.output() {
629 let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
630 return (TrFailed, err.into_bytes(), None);
633 let exec_time = start.map(|start| {
634 let duration = start.elapsed();
635 TestExecTime(duration)
638 let std::process::Output { stdout, stderr, status } = output;
639 let mut test_output = stdout;
640 formatters::write_stderr_delimiter(&mut test_output, &desc.name);
641 test_output.extend_from_slice(&stderr);
643 let result = match (|| -> Result<TestResult, String> {
644 let exit_code = get_exit_code(status)?;
645 Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
649 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
654 (result, test_output, exec_time)
657 let message = CompletedTest::new(desc, result, exec_time, test_output);
658 monitor_ch.send(message).unwrap();
661 fn run_test_in_spawned_subprocess(desc: TestDesc, testfn: Box<dyn FnOnce() + Send>) -> ! {
662 let builtin_panic_hook = panic::take_hook();
663 let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
664 let test_result = match panic_info {
665 Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
666 None => calc_result(&desc, Ok(()), &None, &None),
669 // We don't support serializing TrFailedMsg, so just
670 // print the message out to stderr.
671 if let TrFailedMsg(msg) = &test_result {
672 eprintln!("{}", msg);
675 if let Some(info) = panic_info {
676 builtin_panic_hook(info);
679 if let TrOk = test_result {
680 process::exit(test_result::TR_OK);
682 process::exit(test_result::TR_FAILED);
685 let record_result2 = record_result.clone();
686 panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
689 unreachable!("panic=abort callback should have exited the process")