1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more
12 // Currently, not much of this is meant for users. It is intended to
13 // support the simplest interface possible for representing and
14 // running tests while providing a base that other test frameworks may
17 #![unstable(feature = "test", issue = "50297")]
18 #![doc(test(attr(deny(warnings))))]
19 #![feature(internal_output_capture)]
20 #![feature(is_terminal)]
21 #![feature(staged_api)]
22 #![feature(process_exitcode_internals)]
23 #![feature(panic_can_unwind)]
27 pub use self::bench::{black_box, Bencher};
28 pub use self::console::run_tests_console;
29 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
30 pub use self::types::TestName::*;
31 pub use self::types::*;
32 pub use self::ColorConfig::*;
33 pub use cli::TestOpts;
35 // Module to be used by rustc to compile tests in libtest
40 cli::{parse_opts, TestOpts},
42 helpers::metrics::{Metric, MetricMap},
43 options::{Options, RunIgnored, RunStrategy, ShouldPanic},
44 run_test, test_main, test_main_static,
45 test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
46 time::{TestExecTime, TestTimeOptions},
48 DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc,
49 TestDescAndFn, TestId, TestName, TestType,
55 collections::VecDeque,
59 panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
60 process::{self, Command, Termination},
61 sync::mpsc::{channel, Sender},
64 time::{Duration, Instant},
84 use event::{CompletedTest, TestEvent};
85 use helpers::concurrency::get_concurrency;
86 use helpers::exit_code::get_exit_code;
87 use helpers::shuffle::{get_shuffle_seed, shuffle_tests};
88 use options::RunStrategy;
90 use time::TestExecTime;
92 // Process exit code to be used to indicate test failures.
93 const ERROR_EXIT_CODE: i32 = 101;
95 const SECONDARY_TEST_INVOKER_VAR: &str = "__RUST_TEST_INVOKE";
97 // The default console test runner. It accepts the command line
98 // arguments and a vector of test_descs.
99 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
100 let mut opts = match cli::parse_opts(args) {
103 eprintln!("error: {msg}");
104 process::exit(ERROR_EXIT_CODE);
108 if let Some(options) = options {
109 opts.options = options;
112 if let Err(e) = console::list_tests_console(&opts, tests) {
113 eprintln!("error: io error when listing tests: {e:?}");
114 process::exit(ERROR_EXIT_CODE);
118 // If we encounter a non-unwinding panic, flush any captured output from the current test,
119 // and stop capturing output to ensure that the non-unwinding panic message is visible.
120 // We also acquire the locks for both output streams to prevent output from other threads
121 // from interleaving with the panic message or appearing after it.
122 let builtin_panic_hook = panic::take_hook();
123 let hook = Box::new({
124 move |info: &'_ PanicInfo<'_>| {
125 if !info.can_unwind() {
126 std::mem::forget(std::io::stderr().lock());
127 let mut stdout = ManuallyDrop::new(std::io::stdout().lock());
128 if let Some(captured) = io::set_output_capture(None) {
129 if let Ok(data) = captured.lock() {
130 let _ = stdout.write_all(&data);
131 let _ = stdout.flush();
135 builtin_panic_hook(info);
138 panic::set_hook(hook);
140 match console::run_tests_console(&opts, tests) {
142 Ok(false) => process::exit(ERROR_EXIT_CODE),
144 eprintln!("error: io error when listing tests: {e:?}");
145 process::exit(ERROR_EXIT_CODE);
151 /// A variant optimized for invocation with a static test vector.
152 /// This will panic (intentionally) when fed any dynamic tests.
154 /// This is the entry point for the main function generated by `rustc --test`
155 /// when panic=unwind.
156 pub fn test_main_static(tests: &[&TestDescAndFn]) {
157 let args = env::args().collect::<Vec<_>>();
158 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
159 test_main(&args, owned_tests, None)
162 /// A variant optimized for invocation with a static test vector.
163 /// This will panic (intentionally) when fed any dynamic tests.
165 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
168 /// This is the entry point for the main function generated by `rustc --test`
169 /// when panic=abort.
170 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
171 // If we're being run in SpawnedSecondary mode, run the test here. run_test
172 // will then exit the process.
173 if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
174 env::remove_var(SECONDARY_TEST_INVOKER_VAR);
177 .filter(|test| test.desc.name.as_slice() == name)
178 .map(make_owned_test)
180 .unwrap_or_else(|| panic!("couldn't find a test with the provided name '{name}'"));
181 let TestDescAndFn { desc, testfn } = test;
182 let testfn = match testfn {
183 StaticTestFn(f) => f,
184 _ => panic!("only static tests are supported"),
186 run_test_in_spawned_subprocess(desc, Box::new(testfn));
189 let args = env::args().collect::<Vec<_>>();
190 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
191 test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
194 /// Clones static values for putting into a dynamic vector, which test_main()
195 /// needs to hand out ownership of tests to parallel test runners.
197 /// This will panic when fed any dynamic tests, because they cannot be cloned.
198 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
200 StaticTestFn(f) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() },
201 StaticBenchFn(f) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() },
202 _ => panic!("non-static tests passed to test::test_main_static"),
206 /// Invoked when unit tests terminate. Returns `Result::Err` if the test is
207 /// considered a failure. By default, invokes `report() and checks for a `0`
209 pub fn assert_test_result<T: Termination>(result: T) -> Result<(), String> {
210 let code = result.report().to_i32();
215 "the test returned a termination value with a non-zero status code \
216 ({code}) which indicates a failure"
221 struct FilteredTests {
222 tests: Vec<(TestId, TestDescAndFn)>,
223 benchs: Vec<(TestId, TestDescAndFn)>,
228 fn add_bench(&mut self, desc: TestDesc, testfn: TestFn) {
229 let test = TestDescAndFn { desc, testfn };
230 self.benchs.push((TestId(self.next_id), test));
233 fn add_test(&mut self, desc: TestDesc, testfn: TestFn) {
234 let test = TestDescAndFn { desc, testfn };
235 self.tests.push((TestId(self.next_id), test));
238 fn add_bench_as_test(
241 benchfn: impl Fn(&mut Bencher) -> Result<(), String> + Send + 'static,
243 let testfn = DynTestFn(Box::new(move || {
244 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
246 self.add_test(desc, testfn);
248 fn total_len(&self) -> usize {
249 self.tests.len() + self.benchs.len()
255 tests: Vec<TestDescAndFn>,
256 mut notify_about_test_event: F,
259 F: FnMut(TestEvent) -> io::Result<()>,
261 use std::collections::{self, HashMap};
262 use std::hash::BuildHasherDefault;
263 use std::sync::mpsc::RecvTimeoutError;
266 join_handle: Option<thread::JoinHandle<()>>,
270 fn join(self, completed_test: &mut CompletedTest) {
271 if let Some(join_handle) = self.join_handle {
272 if let Err(_) = join_handle.join() {
273 if let TrOk = completed_test.result {
274 completed_test.result =
275 TrFailedMsg("panicked after reporting success".to_string());
282 // Use a deterministic hasher
284 HashMap<TestId, RunningTest, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
286 struct TimeoutEntry {
292 let tests_len = tests.len();
294 let mut filtered = FilteredTests { tests: Vec::new(), benchs: Vec::new(), next_id: 0 };
296 for test in filter_tests(opts, tests) {
297 let mut desc = test.desc;
298 desc.name = desc.name.with_padding(test.testfn.padding());
301 DynBenchFn(benchfn) => {
302 if opts.bench_benchmarks {
303 filtered.add_bench(desc, DynBenchFn(benchfn));
305 filtered.add_bench_as_test(desc, benchfn);
308 StaticBenchFn(benchfn) => {
309 if opts.bench_benchmarks {
310 filtered.add_bench(desc, StaticBenchFn(benchfn));
312 filtered.add_bench_as_test(desc, benchfn);
316 filtered.add_test(desc, testfn);
321 let filtered_out = tests_len - filtered.total_len();
322 let event = TestEvent::TeFilteredOut(filtered_out);
323 notify_about_test_event(event)?;
325 let shuffle_seed = get_shuffle_seed(opts);
327 let event = TestEvent::TeFiltered(filtered.total_len(), shuffle_seed);
328 notify_about_test_event(event)?;
330 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
332 let mut remaining = filtered.tests;
333 if let Some(shuffle_seed) = shuffle_seed {
334 shuffle_tests(shuffle_seed, &mut remaining);
336 // Store the tests in a VecDeque so we can efficiently remove the first element to run the
337 // tests in the order they were passed (unless shuffled).
338 let mut remaining = VecDeque::from(remaining);
341 let (tx, rx) = channel::<CompletedTest>();
342 let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
343 RunStrategy::SpawnPrimary
345 RunStrategy::InProcess
348 let mut running_tests: TestMap = HashMap::default();
349 let mut timeout_queue: VecDeque<TimeoutEntry> = VecDeque::new();
351 fn get_timed_out_tests(
352 running_tests: &TestMap,
353 timeout_queue: &mut VecDeque<TimeoutEntry>,
355 let now = Instant::now();
356 let mut timed_out = Vec::new();
357 while let Some(timeout_entry) = timeout_queue.front() {
358 if now < timeout_entry.timeout {
361 let timeout_entry = timeout_queue.pop_front().unwrap();
362 if running_tests.contains_key(&timeout_entry.id) {
363 timed_out.push(timeout_entry.desc);
369 fn calc_timeout(timeout_queue: &VecDeque<TimeoutEntry>) -> Option<Duration> {
370 timeout_queue.front().map(|&TimeoutEntry { timeout: next_timeout, .. }| {
371 let now = Instant::now();
372 if next_timeout >= now { next_timeout - now } else { Duration::new(0, 0) }
376 if concurrency == 1 {
377 while !remaining.is_empty() {
378 let (id, test) = remaining.pop_front().unwrap();
379 let event = TestEvent::TeWait(test.desc.clone());
380 notify_about_test_event(event)?;
381 let join_handle = run_test(opts, !opts.run_tests, id, test, run_strategy, tx.clone());
382 // Wait for the test to complete.
383 let mut completed_test = rx.recv().unwrap();
384 RunningTest { join_handle }.join(&mut completed_test);
386 let fail_fast = match completed_test.result {
387 TrIgnored | TrOk | TrBench(_) => false,
388 TrFailed | TrFailedMsg(_) | TrTimedFail => opts.fail_fast,
391 let event = TestEvent::TeResult(completed_test);
392 notify_about_test_event(event)?;
399 while pending > 0 || !remaining.is_empty() {
400 while pending < concurrency && !remaining.is_empty() {
401 let (id, test) = remaining.pop_front().unwrap();
402 let timeout = time::get_default_test_timeout();
403 let desc = test.desc.clone();
405 let event = TestEvent::TeWait(desc.clone());
406 notify_about_test_event(event)?; //here no pad
408 run_test(opts, !opts.run_tests, id, test, run_strategy, tx.clone());
409 running_tests.insert(id, RunningTest { join_handle });
410 timeout_queue.push_back(TimeoutEntry { id, desc, timeout });
416 if let Some(timeout) = calc_timeout(&timeout_queue) {
417 res = rx.recv_timeout(timeout);
418 for test in get_timed_out_tests(&running_tests, &mut timeout_queue) {
419 let event = TestEvent::TeTimeout(test);
420 notify_about_test_event(event)?;
424 Err(RecvTimeoutError::Timeout) => {
425 // Result is not yet ready, continue waiting.
428 // We've got a result, stop the loop.
433 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
438 let mut completed_test = res.unwrap();
439 let running_test = running_tests.remove(&completed_test.id).unwrap();
440 running_test.join(&mut completed_test);
442 let fail_fast = match completed_test.result {
443 TrIgnored | TrOk | TrBench(_) => false,
444 TrFailed | TrFailedMsg(_) | TrTimedFail => opts.fail_fast,
447 let event = TestEvent::TeResult(completed_test);
448 notify_about_test_event(event)?;
452 // Prevent remaining test threads from panicking
453 std::mem::forget(rx);
459 if opts.bench_benchmarks {
460 // All benchmarks run at the end, in serial.
461 for (id, b) in filtered.benchs {
462 let event = TestEvent::TeWait(b.desc.clone());
463 notify_about_test_event(event)?;
464 let join_handle = run_test(opts, false, id, b, run_strategy, tx.clone());
465 // Wait for the test to complete.
466 let mut completed_test = rx.recv().unwrap();
467 RunningTest { join_handle }.join(&mut completed_test);
469 let event = TestEvent::TeResult(completed_test);
470 notify_about_test_event(event)?;
476 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
477 let mut filtered = tests;
478 let matches_filter = |test: &TestDescAndFn, filter: &str| {
479 let test_name = test.desc.name.as_slice();
481 match opts.filter_exact {
482 true => test_name == filter,
483 false => test_name.contains(filter),
487 // Remove tests that don't match the test filter
488 if !opts.filters.is_empty() {
489 filtered.retain(|test| opts.filters.iter().any(|filter| matches_filter(test, filter)));
492 // Skip tests that match any of the skip filters
493 if !opts.skip.is_empty() {
494 filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
497 // Excludes #[should_panic] tests
498 if opts.exclude_should_panic {
499 filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
502 // maybe unignore tests
503 match opts.run_ignored {
505 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
507 RunIgnored::Only => {
508 filtered.retain(|test| test.desc.ignore);
509 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
517 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
518 // convert benchmarks to tests, if we're not benchmarking them
522 let testfn = match x.testfn {
523 DynBenchFn(benchfn) => DynTestFn(Box::new(move || {
524 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
526 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
527 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
531 TestDescAndFn { desc: x.desc, testfn }
541 strategy: RunStrategy,
542 monitor_ch: Sender<CompletedTest>,
543 ) -> Option<thread::JoinHandle<()>> {
544 let TestDescAndFn { desc, testfn } = test;
546 // Emscripten can catch panics but other wasm targets cannot
547 let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
548 && cfg!(target_family = "wasm")
549 && !cfg!(target_os = "emscripten");
551 if force_ignore || desc.ignore || ignore_because_no_process_support {
552 let message = CompletedTest::new(id, desc, TrIgnored, None, Vec::new());
553 monitor_ch.send(message).unwrap();
558 pub strategy: RunStrategy,
560 pub time: Option<time::TestTimeOptions>,
566 monitor_ch: Sender<CompletedTest>,
567 testfn: Box<dyn FnOnce() -> Result<(), String> + Send>,
569 ) -> Option<thread::JoinHandle<()>> {
570 let name = desc.name.clone();
572 let runtest = move || match opts.strategy {
573 RunStrategy::InProcess => run_test_in_process(
582 RunStrategy::SpawnPrimary => spawn_test_subprocess(
592 // If the platform is single-threaded we're just going to run
593 // the test synchronously, regardless of the concurrency
595 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_family = "wasm");
596 if supports_threads {
597 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
598 let mut runtest = Arc::new(Mutex::new(Some(runtest)));
599 let runtest2 = runtest.clone();
600 match cfg.spawn(move || runtest2.lock().unwrap().take().unwrap()()) {
601 Ok(handle) => Some(handle),
602 Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
603 // `ErrorKind::WouldBlock` means hitting the thread limit on some
604 // platforms, so run the test synchronously here instead.
605 Arc::get_mut(&mut runtest).unwrap().get_mut().unwrap().take().unwrap()();
608 Err(e) => panic!("failed to spawn thread to run test: {e}"),
617 TestRunOpts { strategy, nocapture: opts.nocapture, time: opts.time_options };
620 DynBenchFn(benchfn) => {
621 // Benchmarks aren't expected to panic, so we run them all in-process.
622 crate::bench::benchmark(id, desc, monitor_ch, opts.nocapture, benchfn);
625 StaticBenchFn(benchfn) => {
626 // Benchmarks aren't expected to panic, so we run them all in-process.
627 crate::bench::benchmark(id, desc, monitor_ch, opts.nocapture, benchfn);
632 RunStrategy::InProcess => (),
633 _ => panic!("Cannot run dynamic test fn out-of-process"),
639 Box::new(move || __rust_begin_short_backtrace(f)),
643 StaticTestFn(f) => run_test_inner(
647 Box::new(move || __rust_begin_short_backtrace(f)),
653 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
655 fn __rust_begin_short_backtrace<T, F: FnOnce() -> T>(f: F) -> T {
658 // prevent this frame from being tail-call optimised away
662 fn run_test_in_process(
667 testfn: Box<dyn FnOnce() -> Result<(), String> + Send>,
668 monitor_ch: Sender<CompletedTest>,
669 time_opts: Option<time::TestTimeOptions>,
671 // Buffer for capturing standard I/O
672 let data = Arc::new(Mutex::new(Vec::new()));
675 io::set_output_capture(Some(data.clone()));
678 let start = report_time.then(Instant::now);
679 let result = fold_err(catch_unwind(AssertUnwindSafe(testfn)));
680 let exec_time = start.map(|start| {
681 let duration = start.elapsed();
682 TestExecTime(duration)
685 io::set_output_capture(None);
687 let test_result = match result {
688 Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
689 Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
691 let stdout = data.lock().unwrap_or_else(|e| e.into_inner()).to_vec();
692 let message = CompletedTest::new(id, desc, test_result, exec_time, stdout);
693 monitor_ch.send(message).unwrap();
697 result: Result<Result<T, E>, Box<dyn Any + Send>>,
698 ) -> Result<T, Box<dyn Any + Send>>
703 Ok(Err(e)) => Err(Box::new(e)),
709 fn spawn_test_subprocess(
714 monitor_ch: Sender<CompletedTest>,
715 time_opts: Option<time::TestTimeOptions>,
717 let (result, test_output, exec_time) = (|| {
718 let args = env::args().collect::<Vec<_>>();
719 let current_exe = &args[0];
721 let mut command = Command::new(current_exe);
722 command.env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice());
724 command.stdout(process::Stdio::inherit());
725 command.stderr(process::Stdio::inherit());
728 let start = report_time.then(Instant::now);
729 let output = match command.output() {
732 let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
733 return (TrFailed, err.into_bytes(), None);
736 let exec_time = start.map(|start| {
737 let duration = start.elapsed();
738 TestExecTime(duration)
741 let std::process::Output { stdout, stderr, status } = output;
742 let mut test_output = stdout;
743 formatters::write_stderr_delimiter(&mut test_output, &desc.name);
744 test_output.extend_from_slice(&stderr);
746 let result = match (|| -> Result<TestResult, String> {
747 let exit_code = get_exit_code(status)?;
748 Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
752 write!(&mut test_output, "Unexpected error: {e}").unwrap();
757 (result, test_output, exec_time)
760 let message = CompletedTest::new(id, desc, result, exec_time, test_output);
761 monitor_ch.send(message).unwrap();
764 fn run_test_in_spawned_subprocess(
766 testfn: Box<dyn FnOnce() -> Result<(), String> + Send>,
768 let builtin_panic_hook = panic::take_hook();
769 let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
770 let test_result = match panic_info {
771 Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
772 None => calc_result(&desc, Ok(()), &None, &None),
775 // We don't support serializing TrFailedMsg, so just
776 // print the message out to stderr.
777 if let TrFailedMsg(msg) = &test_result {
781 if let Some(info) = panic_info {
782 builtin_panic_hook(info);
785 if let TrOk = test_result {
786 process::exit(test_result::TR_OK);
788 process::exit(test_result::TR_FAILED);
791 let record_result2 = record_result.clone();
792 panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
793 if let Err(message) = testfn() {
794 panic!("{}", message);
797 unreachable!("panic=abort callback should have exited the process")