1 //! Support code for rustc's built in unit-test and micro-benchmarking
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more
12 // Currently, not much of this is meant for users. It is intended to
13 // support the simplest interface possible for representing and
14 // running tests while providing a base that other test frameworks may
17 #![unstable(feature = "test", issue = "50297")]
18 #![doc(test(attr(deny(warnings))))]
19 #![feature(internal_output_capture)]
20 #![feature(staged_api)]
21 #![feature(process_exitcode_internals)]
25 pub use self::bench::{black_box, Bencher};
26 pub use self::console::run_tests_console;
27 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
28 pub use self::types::TestName::*;
29 pub use self::types::*;
30 pub use self::ColorConfig::*;
31 pub use cli::TestOpts;
33 // Module to be used by rustc to compile tests in libtest
38 cli::{parse_opts, TestOpts},
40 helpers::metrics::{Metric, MetricMap},
41 options::{Concurrent, Options, RunIgnored, RunStrategy, ShouldPanic},
42 run_test, test_main, test_main_static,
43 test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
44 time::{TestExecTime, TestTimeOptions},
46 DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc,
47 TestDescAndFn, TestId, TestName, TestType,
53 collections::VecDeque,
56 panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
57 process::{self, Command, Termination},
58 sync::mpsc::{channel, Sender},
61 time::{Duration, Instant},
81 use event::{CompletedTest, TestEvent};
82 use helpers::concurrency::get_concurrency;
83 use helpers::exit_code::get_exit_code;
84 use helpers::shuffle::{get_shuffle_seed, shuffle_tests};
85 use options::{Concurrent, RunStrategy};
87 use time::TestExecTime;
89 // Process exit code to be used to indicate test failures.
90 const ERROR_EXIT_CODE: i32 = 101;
92 const SECONDARY_TEST_INVOKER_VAR: &str = "__RUST_TEST_INVOKE";
94 // The default console test runner. It accepts the command line
95 // arguments and a vector of test_descs.
96 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
97 let mut opts = match cli::parse_opts(args) {
100 eprintln!("error: {msg}");
101 process::exit(ERROR_EXIT_CODE);
105 if let Some(options) = options {
106 opts.options = options;
109 if let Err(e) = console::list_tests_console(&opts, tests) {
110 eprintln!("error: io error when listing tests: {e:?}");
111 process::exit(ERROR_EXIT_CODE);
114 match console::run_tests_console(&opts, tests) {
116 Ok(false) => process::exit(ERROR_EXIT_CODE),
118 eprintln!("error: io error when listing tests: {e:?}");
119 process::exit(ERROR_EXIT_CODE);
125 /// A variant optimized for invocation with a static test vector.
126 /// This will panic (intentionally) when fed any dynamic tests.
128 /// This is the entry point for the main function generated by `rustc --test`
129 /// when panic=unwind.
130 pub fn test_main_static(tests: &[&TestDescAndFn]) {
131 let args = env::args().collect::<Vec<_>>();
132 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
133 test_main(&args, owned_tests, None)
136 /// A variant optimized for invocation with a static test vector.
137 /// This will panic (intentionally) when fed any dynamic tests.
139 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
142 /// This is the entry point for the main function generated by `rustc --test`
143 /// when panic=abort.
144 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
145 // If we're being run in SpawnedSecondary mode, run the test here. run_test
146 // will then exit the process.
147 if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
148 env::remove_var(SECONDARY_TEST_INVOKER_VAR);
151 .filter(|test| test.desc.name.as_slice() == name)
152 .map(make_owned_test)
154 .unwrap_or_else(|| panic!("couldn't find a test with the provided name '{name}'"));
155 let TestDescAndFn { desc, testfn } = test;
156 let testfn = match testfn {
157 StaticTestFn(f) => f,
158 _ => panic!("only static tests are supported"),
160 run_test_in_spawned_subprocess(desc, Box::new(testfn));
163 let args = env::args().collect::<Vec<_>>();
164 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
165 test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
168 /// Clones static values for putting into a dynamic vector, which test_main()
169 /// needs to hand out ownership of tests to parallel test runners.
171 /// This will panic when fed any dynamic tests, because they cannot be cloned.
172 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
174 StaticTestFn(f) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() },
175 StaticBenchFn(f) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() },
176 _ => panic!("non-static tests passed to test::test_main_static"),
180 /// Invoked when unit tests terminate. Returns `Result::Err` if the test is
181 /// considered a failure. By default, invokes `report() and checks for a `0`
183 pub fn assert_test_result<T: Termination>(result: T) -> Result<(), String> {
184 let code = result.report().to_i32();
189 "the test returned a termination value with a non-zero status code \
190 ({}) which indicates a failure",
198 tests: Vec<TestDescAndFn>,
199 mut notify_about_test_event: F,
202 F: FnMut(TestEvent) -> io::Result<()>,
204 use std::collections::{self, HashMap};
205 use std::hash::BuildHasherDefault;
206 use std::sync::mpsc::RecvTimeoutError;
209 join_handle: Option<thread::JoinHandle<()>>,
212 // Use a deterministic hasher
214 HashMap<TestId, RunningTest, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
216 struct TimeoutEntry {
222 let tests_len = tests.len();
224 let mut filtered_tests = filter_tests(opts, tests);
225 if !opts.bench_benchmarks {
226 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
229 let filtered_tests = {
230 let mut filtered_tests = filtered_tests;
231 for test in filtered_tests.iter_mut() {
232 test.desc.name = test.desc.name.with_padding(test.testfn.padding());
238 let filtered_out = tests_len - filtered_tests.len();
239 let event = TestEvent::TeFilteredOut(filtered_out);
240 notify_about_test_event(event)?;
242 let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
244 let shuffle_seed = get_shuffle_seed(opts);
246 let event = TestEvent::TeFiltered(filtered_descs, shuffle_seed);
247 notify_about_test_event(event)?;
249 let (filtered_tests, filtered_benchs): (Vec<_>, _) = filtered_tests
252 .map(|(i, e)| (TestId(i), e))
253 .partition(|(_, e)| matches!(e.testfn, StaticTestFn(_) | DynTestFn(_)));
255 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
257 let mut remaining = filtered_tests;
258 if let Some(shuffle_seed) = shuffle_seed {
259 shuffle_tests(shuffle_seed, &mut remaining);
265 let (tx, rx) = channel::<CompletedTest>();
266 let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
267 RunStrategy::SpawnPrimary
269 RunStrategy::InProcess
272 let mut running_tests: TestMap = HashMap::default();
273 let mut timeout_queue: VecDeque<TimeoutEntry> = VecDeque::new();
275 fn get_timed_out_tests(
276 running_tests: &TestMap,
277 timeout_queue: &mut VecDeque<TimeoutEntry>,
279 let now = Instant::now();
280 let mut timed_out = Vec::new();
281 while let Some(timeout_entry) = timeout_queue.front() {
282 if now < timeout_entry.timeout {
285 let timeout_entry = timeout_queue.pop_front().unwrap();
286 if running_tests.contains_key(&timeout_entry.id) {
287 timed_out.push(timeout_entry.desc);
293 fn calc_timeout(timeout_queue: &VecDeque<TimeoutEntry>) -> Option<Duration> {
294 timeout_queue.front().map(|&TimeoutEntry { timeout: next_timeout, .. }| {
295 let now = Instant::now();
296 if next_timeout >= now { next_timeout - now } else { Duration::new(0, 0) }
300 if concurrency == 1 {
301 while !remaining.is_empty() {
302 let (id, test) = remaining.pop().unwrap();
303 let event = TestEvent::TeWait(test.desc.clone());
304 notify_about_test_event(event)?;
306 run_test(opts, !opts.run_tests, id, test, run_strategy, tx.clone(), Concurrent::No);
307 assert!(join_handle.is_none());
308 let completed_test = rx.recv().unwrap();
310 let event = TestEvent::TeResult(completed_test);
311 notify_about_test_event(event)?;
314 while pending > 0 || !remaining.is_empty() {
315 while pending < concurrency && !remaining.is_empty() {
316 let (id, test) = remaining.pop().unwrap();
317 let timeout = time::get_default_test_timeout();
318 let desc = test.desc.clone();
320 let event = TestEvent::TeWait(desc.clone());
321 notify_about_test_event(event)?; //here no pad
322 let join_handle = run_test(
331 running_tests.insert(id, RunningTest { join_handle });
332 timeout_queue.push_back(TimeoutEntry { id, desc, timeout });
338 if let Some(timeout) = calc_timeout(&timeout_queue) {
339 res = rx.recv_timeout(timeout);
340 for test in get_timed_out_tests(&running_tests, &mut timeout_queue) {
341 let event = TestEvent::TeTimeout(test);
342 notify_about_test_event(event)?;
346 Err(RecvTimeoutError::Timeout) => {
347 // Result is not yet ready, continue waiting.
350 // We've got a result, stop the loop.
355 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
360 let mut completed_test = res.unwrap();
361 let running_test = running_tests.remove(&completed_test.id).unwrap();
362 if let Some(join_handle) = running_test.join_handle {
363 if let Err(_) = join_handle.join() {
364 if let TrOk = completed_test.result {
365 completed_test.result =
366 TrFailedMsg("panicked after reporting success".to_string());
371 let event = TestEvent::TeResult(completed_test);
372 notify_about_test_event(event)?;
377 if opts.bench_benchmarks {
378 // All benchmarks run at the end, in serial.
379 for (id, b) in filtered_benchs {
380 let event = TestEvent::TeWait(b.desc.clone());
381 notify_about_test_event(event)?;
382 run_test(opts, false, id, b, run_strategy, tx.clone(), Concurrent::No);
383 let completed_test = rx.recv().unwrap();
385 let event = TestEvent::TeResult(completed_test);
386 notify_about_test_event(event)?;
392 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
393 let mut filtered = tests;
394 let matches_filter = |test: &TestDescAndFn, filter: &str| {
395 let test_name = test.desc.name.as_slice();
397 match opts.filter_exact {
398 true => test_name == filter,
399 false => test_name.contains(filter),
403 // Remove tests that don't match the test filter
404 if !opts.filters.is_empty() {
405 filtered.retain(|test| opts.filters.iter().any(|filter| matches_filter(test, filter)));
408 // Skip tests that match any of the skip filters
409 filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
411 // Excludes #[should_panic] tests
412 if opts.exclude_should_panic {
413 filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
416 // maybe unignore tests
417 match opts.run_ignored {
419 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
421 RunIgnored::Only => {
422 filtered.retain(|test| test.desc.ignore);
423 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
428 // Sort the tests alphabetically
429 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
434 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
435 // convert benchmarks to tests, if we're not benchmarking them
439 let testfn = match x.testfn {
440 DynBenchFn(benchfn) => DynTestFn(Box::new(move || {
441 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
443 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
444 bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
448 TestDescAndFn { desc: x.desc, testfn }
458 strategy: RunStrategy,
459 monitor_ch: Sender<CompletedTest>,
460 concurrency: Concurrent,
461 ) -> Option<thread::JoinHandle<()>> {
462 let TestDescAndFn { desc, testfn } = test;
464 // Emscripten can catch panics but other wasm targets cannot
465 let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
466 && cfg!(target_family = "wasm")
467 && !cfg!(target_os = "emscripten");
469 if force_ignore || desc.ignore || ignore_because_no_process_support {
470 let message = CompletedTest::new(id, desc, TrIgnored, None, Vec::new());
471 monitor_ch.send(message).unwrap();
476 pub strategy: RunStrategy,
478 pub concurrency: Concurrent,
479 pub time: Option<time::TestTimeOptions>,
485 monitor_ch: Sender<CompletedTest>,
486 testfn: Box<dyn FnOnce() -> Result<(), String> + Send>,
488 ) -> Option<thread::JoinHandle<()>> {
489 let concurrency = opts.concurrency;
490 let name = desc.name.clone();
492 let runtest = move || match opts.strategy {
493 RunStrategy::InProcess => run_test_in_process(
502 RunStrategy::SpawnPrimary => spawn_test_subprocess(
512 // If the platform is single-threaded we're just going to run
513 // the test synchronously, regardless of the concurrency
515 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_family = "wasm");
516 if concurrency == Concurrent::Yes && supports_threads {
517 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
518 let mut runtest = Arc::new(Mutex::new(Some(runtest)));
519 let runtest2 = runtest.clone();
520 match cfg.spawn(move || runtest2.lock().unwrap().take().unwrap()()) {
521 Ok(handle) => Some(handle),
522 Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
523 // `ErrorKind::WouldBlock` means hitting the thread limit on some
524 // platforms, so run the test synchronously here instead.
525 Arc::get_mut(&mut runtest).unwrap().get_mut().unwrap().take().unwrap()();
528 Err(e) => panic!("failed to spawn thread to run test: {e}"),
537 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options };
540 DynBenchFn(benchfn) => {
541 // Benchmarks aren't expected to panic, so we run them all in-process.
542 crate::bench::benchmark(id, desc, monitor_ch, opts.nocapture, benchfn);
545 StaticBenchFn(benchfn) => {
546 // Benchmarks aren't expected to panic, so we run them all in-process.
547 crate::bench::benchmark(id, desc, monitor_ch, opts.nocapture, benchfn);
552 RunStrategy::InProcess => (),
553 _ => panic!("Cannot run dynamic test fn out-of-process"),
559 Box::new(move || __rust_begin_short_backtrace(f)),
563 StaticTestFn(f) => run_test_inner(
567 Box::new(move || __rust_begin_short_backtrace(f)),
573 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
575 fn __rust_begin_short_backtrace<T, F: FnOnce() -> T>(f: F) -> T {
578 // prevent this frame from being tail-call optimised away
582 fn run_test_in_process(
587 testfn: Box<dyn FnOnce() -> Result<(), String> + Send>,
588 monitor_ch: Sender<CompletedTest>,
589 time_opts: Option<time::TestTimeOptions>,
591 // Buffer for capturing standard I/O
592 let data = Arc::new(Mutex::new(Vec::new()));
595 io::set_output_capture(Some(data.clone()));
598 let start = report_time.then(Instant::now);
599 let result = fold_err(catch_unwind(AssertUnwindSafe(testfn)));
600 let exec_time = start.map(|start| {
601 let duration = start.elapsed();
602 TestExecTime(duration)
605 io::set_output_capture(None);
607 let test_result = match result {
608 Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
609 Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
611 let stdout = data.lock().unwrap_or_else(|e| e.into_inner()).to_vec();
612 let message = CompletedTest::new(id, desc, test_result, exec_time, stdout);
613 monitor_ch.send(message).unwrap();
617 result: Result<Result<T, E>, Box<dyn Any + Send>>,
618 ) -> Result<T, Box<dyn Any + Send>>
623 Ok(Err(e)) => Err(Box::new(e)),
629 fn spawn_test_subprocess(
634 monitor_ch: Sender<CompletedTest>,
635 time_opts: Option<time::TestTimeOptions>,
637 let (result, test_output, exec_time) = (|| {
638 let args = env::args().collect::<Vec<_>>();
639 let current_exe = &args[0];
641 let mut command = Command::new(current_exe);
642 command.env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice());
644 command.stdout(process::Stdio::inherit());
645 command.stderr(process::Stdio::inherit());
648 let start = report_time.then(Instant::now);
649 let output = match command.output() {
652 let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
653 return (TrFailed, err.into_bytes(), None);
656 let exec_time = start.map(|start| {
657 let duration = start.elapsed();
658 TestExecTime(duration)
661 let std::process::Output { stdout, stderr, status } = output;
662 let mut test_output = stdout;
663 formatters::write_stderr_delimiter(&mut test_output, &desc.name);
664 test_output.extend_from_slice(&stderr);
666 let result = match (|| -> Result<TestResult, String> {
667 let exit_code = get_exit_code(status)?;
668 Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
672 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
677 (result, test_output, exec_time)
680 let message = CompletedTest::new(id, desc, result, exec_time, test_output);
681 monitor_ch.send(message).unwrap();
684 fn run_test_in_spawned_subprocess(
686 testfn: Box<dyn FnOnce() -> Result<(), String> + Send>,
688 let builtin_panic_hook = panic::take_hook();
689 let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
690 let test_result = match panic_info {
691 Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
692 None => calc_result(&desc, Ok(()), &None, &None),
695 // We don't support serializing TrFailedMsg, so just
696 // print the message out to stderr.
697 if let TrFailedMsg(msg) = &test_result {
701 if let Some(info) = panic_info {
702 builtin_panic_hook(info);
705 if let TrOk = test_result {
706 process::exit(test_result::TR_OK);
708 process::exit(test_result::TR_FAILED);
711 let record_result2 = record_result.clone();
712 panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
713 if let Err(message) = testfn() {
714 panic!("{}", message);
717 unreachable!("panic=abort callback should have exited the process")