]> git.lizzy.rs Git - rust.git/blob - src/libtest/lib.rs
Rollup merge of #67015 - osa1:issue66971, r=wesleywiser
[rust.git] / src / libtest / lib.rs
1 //! Support code for rustc's built in unit-test and micro-benchmarking
2 //! framework.
3 //!
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
8 //!
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
10
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
14 // build off of.
15
16 // N.B., this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
19
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
23 #![feature(asm)]
24 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
25 #![feature(rustc_private)]
26 #![feature(nll)]
27 #![feature(bool_to_option)]
28 #![feature(set_stdio)]
29 #![feature(panic_unwind)]
30 #![feature(staged_api)]
31 #![feature(termination_trait_lib)]
32 #![feature(test)]
33
34 // Public reexports
35 pub use self::ColorConfig::*;
36 pub use self::types::*;
37 pub use self::types::TestName::*;
38 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
39 pub use self::bench::{Bencher, black_box};
40 pub use self::console::run_tests_console;
41 pub use cli::TestOpts;
42
43 // Module to be used by rustc to compile tests in libtest
44 pub mod test {
45     pub use crate::{
46         bench::Bencher,
47         cli::{parse_opts, TestOpts},
48         helpers::metrics::{Metric, MetricMap},
49         options::{ShouldPanic, Options, RunIgnored, RunStrategy},
50         test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
51         time::{TestTimeOptions, TestExecTime},
52         types::{
53             DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName,
54             TestDesc, TestDescAndFn, TestName, TestType,
55         },
56         assert_test_result, filter_tests, run_test, test_main, test_main_static,
57     };
58 }
59
60 use std::{
61     env,
62     io,
63     io::prelude::Write,
64     panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
65     process,
66     process::{Command, Termination},
67     sync::mpsc::{channel, Sender},
68     sync::{Arc, Mutex},
69     thread,
70     time::{Duration, Instant},
71 };
72
73 pub mod stats;
74 pub mod bench;
75 mod formatters;
76 mod cli;
77 mod console;
78 mod event;
79 mod helpers;
80 mod time;
81 mod types;
82 mod options;
83 mod test_result;
84
85 #[cfg(test)]
86 mod tests;
87
88 use test_result::*;
89 use time::TestExecTime;
90 use options::{RunStrategy, Concurrent};
91 use event::{CompletedTest, TestEvent};
92 use helpers::sink::Sink;
93 use helpers::concurrency::get_concurrency;
94 use helpers::exit_code::get_exit_code;
95
96 // Process exit code to be used to indicate test failures.
97 const ERROR_EXIT_CODE: i32 = 101;
98
99 const SECONDARY_TEST_INVOKER_VAR: &'static str = "__RUST_TEST_INVOKE";
100
101 // The default console test runner. It accepts the command line
102 // arguments and a vector of test_descs.
103 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
104     let mut opts = match cli::parse_opts(args) {
105         Some(Ok(o)) => o,
106         Some(Err(msg)) => {
107             eprintln!("error: {}", msg);
108             process::exit(ERROR_EXIT_CODE);
109         }
110         None => return,
111     };
112     if let Some(options) = options {
113         opts.options = options;
114     }
115     if opts.list {
116         if let Err(e) = console::list_tests_console(&opts, tests) {
117             eprintln!("error: io error when listing tests: {:?}", e);
118             process::exit(ERROR_EXIT_CODE);
119         }
120     } else {
121         match console::run_tests_console(&opts, tests) {
122             Ok(true) => {}
123             Ok(false) => process::exit(ERROR_EXIT_CODE),
124             Err(e) => {
125                 eprintln!("error: io error when listing tests: {:?}", e);
126                 process::exit(ERROR_EXIT_CODE);
127             }
128         }
129     }
130 }
131
132 /// A variant optimized for invocation with a static test vector.
133 /// This will panic (intentionally) when fed any dynamic tests.
134 ///
135 /// This is the entry point for the main function generated by `rustc --test`
136 /// when panic=unwind.
137 pub fn test_main_static(tests: &[&TestDescAndFn]) {
138     let args = env::args().collect::<Vec<_>>();
139     let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
140     test_main(&args, owned_tests, None)
141 }
142
143 /// A variant optimized for invocation with a static test vector.
144 /// This will panic (intentionally) when fed any dynamic tests.
145 ///
146 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
147 /// tests.
148 ///
149 /// This is the entry point for the main function generated by `rustc --test`
150 /// when panic=abort.
151 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
152     // If we're being run in SpawnedSecondary mode, run the test here. run_test
153     // will then exit the process.
154     if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
155         let test = tests
156             .iter()
157             .filter(|test| test.desc.name.as_slice() == name)
158             .map(make_owned_test)
159             .next()
160             .expect("couldn't find a test with the provided name");
161         let TestDescAndFn { desc, testfn } = test;
162         let testfn = match testfn {
163             StaticTestFn(f) => f,
164             _ => panic!("only static tests are supported"),
165         };
166         run_test_in_spawned_subprocess(desc, Box::new(testfn));
167     }
168
169     let args = env::args().collect::<Vec<_>>();
170     let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
171     test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
172 }
173
174 /// Clones static values for putting into a dynamic vector, which test_main()
175 /// needs to hand out ownership of tests to parallel test runners.
176 ///
177 /// This will panic when fed any dynamic tests, because they cannot be cloned.
178 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
179     match test.testfn {
180         StaticTestFn(f) => TestDescAndFn {
181             testfn: StaticTestFn(f),
182             desc: test.desc.clone(),
183         },
184         StaticBenchFn(f) => TestDescAndFn {
185             testfn: StaticBenchFn(f),
186             desc: test.desc.clone(),
187         },
188         _ => panic!("non-static tests passed to test::test_main_static"),
189     }
190 }
191
192 /// Invoked when unit tests terminate. Should panic if the unit
193 /// Tests is considered a failure. By default, invokes `report()`
194 /// and checks for a `0` result.
195 pub fn assert_test_result<T: Termination>(result: T) {
196     let code = result.report();
197     assert_eq!(
198         code, 0,
199         "the test returned a termination value with a non-zero status code ({}) \
200          which indicates a failure",
201         code
202     );
203 }
204
205 pub fn run_tests<F>(
206     opts: &TestOpts,
207     tests: Vec<TestDescAndFn>,
208     mut notify_about_test_event: F
209 ) -> io::Result<()>
210 where
211     F: FnMut(TestEvent) -> io::Result<()>,
212 {
213     use std::collections::{self, HashMap};
214     use std::hash::BuildHasherDefault;
215     use std::sync::mpsc::RecvTimeoutError;
216     // Use a deterministic hasher
217     type TestMap =
218         HashMap<TestDesc, Instant, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
219
220     let tests_len = tests.len();
221
222     let mut filtered_tests = filter_tests(opts, tests);
223     if !opts.bench_benchmarks {
224         filtered_tests = convert_benchmarks_to_tests(filtered_tests);
225     }
226
227     let filtered_tests = {
228         let mut filtered_tests = filtered_tests;
229         for test in filtered_tests.iter_mut() {
230             test.desc.name = test.desc.name.with_padding(test.testfn.padding());
231         }
232
233         filtered_tests
234     };
235
236     let filtered_out = tests_len - filtered_tests.len();
237     let event = TestEvent::TeFilteredOut(filtered_out);
238     notify_about_test_event(event)?;
239
240     let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
241
242     let event = TestEvent::TeFiltered(filtered_descs);
243     notify_about_test_event(event)?;
244
245     let (filtered_tests, filtered_benchs): (Vec<_>, _) =
246         filtered_tests.into_iter().partition(|e| match e.testfn {
247             StaticTestFn(_) | DynTestFn(_) => true,
248             _ => false,
249         });
250
251     let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
252
253     let mut remaining = filtered_tests;
254     remaining.reverse();
255     let mut pending = 0;
256
257     let (tx, rx) = channel::<CompletedTest>();
258     let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
259         RunStrategy::SpawnPrimary
260     } else {
261         RunStrategy::InProcess
262     };
263
264     let mut running_tests: TestMap = HashMap::default();
265
266     fn get_timed_out_tests(running_tests: &mut TestMap) -> Vec<TestDesc> {
267         let now = Instant::now();
268         let timed_out = running_tests
269             .iter()
270             .filter_map(|(desc, timeout)| {
271                 if &now >= timeout {
272                     Some(desc.clone())
273                 } else {
274                     None
275                 }
276             })
277             .collect();
278         for test in &timed_out {
279             running_tests.remove(test);
280         }
281         timed_out
282     };
283
284     fn calc_timeout(running_tests: &TestMap) -> Option<Duration> {
285         running_tests.values().min().map(|next_timeout| {
286             let now = Instant::now();
287             if *next_timeout >= now {
288                 *next_timeout - now
289             } else {
290                 Duration::new(0, 0)
291             }
292         })
293     };
294
295     if concurrency == 1 {
296         while !remaining.is_empty() {
297             let test = remaining.pop().unwrap();
298             let event = TestEvent::TeWait(test.desc.clone());
299             notify_about_test_event(event)?;
300             run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No);
301             let completed_test = rx.recv().unwrap();
302
303             let event = TestEvent::TeResult(completed_test);
304             notify_about_test_event(event)?;
305         }
306     } else {
307         while pending > 0 || !remaining.is_empty() {
308             while pending < concurrency && !remaining.is_empty() {
309                 let test = remaining.pop().unwrap();
310                 let timeout = time::get_default_test_timeout();
311                 running_tests.insert(test.desc.clone(), timeout);
312
313                 let event = TestEvent::TeWait(test.desc.clone());
314                 notify_about_test_event(event)?; //here no pad
315                 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::Yes);
316                 pending += 1;
317             }
318
319             let mut res;
320             loop {
321                 if let Some(timeout) = calc_timeout(&running_tests) {
322                     res = rx.recv_timeout(timeout);
323                     for test in get_timed_out_tests(&mut running_tests) {
324                         let event = TestEvent::TeTimeout(test);
325                         notify_about_test_event(event)?;
326                     }
327
328                     match res {
329                         Err(RecvTimeoutError::Timeout) => {
330                             // Result is not yet ready, continue waiting.
331                         }
332                         _ => {
333                             // We've got a result, stop the loop.
334                             break;
335                         }
336                     }
337                 } else {
338                     res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
339                     break;
340                 }
341             }
342
343             let completed_test = res.unwrap();
344             running_tests.remove(&completed_test.desc);
345
346             let event = TestEvent::TeResult(completed_test);
347             notify_about_test_event(event)?;
348             pending -= 1;
349         }
350     }
351
352     if opts.bench_benchmarks {
353         // All benchmarks run at the end, in serial.
354         for b in filtered_benchs {
355             let event = TestEvent::TeWait(b.desc.clone());
356             notify_about_test_event(event)?;
357             run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
358             let completed_test = rx.recv().unwrap();
359
360             let event = TestEvent::TeResult(completed_test);
361             notify_about_test_event(event)?;
362         }
363     }
364     Ok(())
365 }
366
367 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
368     let mut filtered = tests;
369     let matches_filter = |test: &TestDescAndFn, filter: &str| {
370         let test_name = test.desc.name.as_slice();
371
372         match opts.filter_exact {
373             true => test_name == filter,
374             false => test_name.contains(filter),
375         }
376     };
377
378     // Remove tests that don't match the test filter
379     if let Some(ref filter) = opts.filter {
380         filtered.retain(|test| matches_filter(test, filter));
381     }
382
383     // Skip tests that match any of the skip filters
384     filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
385
386     // Excludes #[should_panic] tests
387     if opts.exclude_should_panic {
388         filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
389     }
390
391     // maybe unignore tests
392     match opts.run_ignored {
393         RunIgnored::Yes => {
394             filtered
395                 .iter_mut()
396                 .for_each(|test| test.desc.ignore = false);
397         }
398         RunIgnored::Only => {
399             filtered.retain(|test| test.desc.ignore);
400             filtered
401                 .iter_mut()
402                 .for_each(|test| test.desc.ignore = false);
403         }
404         RunIgnored::No => {}
405     }
406
407     // Sort the tests alphabetically
408     filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
409
410     filtered
411 }
412
413 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
414     // convert benchmarks to tests, if we're not benchmarking them
415     tests
416         .into_iter()
417         .map(|x| {
418             let testfn = match x.testfn {
419                 DynBenchFn(bench) => DynTestFn(Box::new(move || {
420                     bench::run_once(|b| __rust_begin_short_backtrace(|| bench.run(b)))
421                 })),
422                 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
423                     bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
424                 })),
425                 f => f,
426             };
427             TestDescAndFn {
428                 desc: x.desc,
429                 testfn,
430             }
431         })
432         .collect()
433 }
434
435 pub fn run_test(
436     opts: &TestOpts,
437     force_ignore: bool,
438     test: TestDescAndFn,
439     strategy: RunStrategy,
440     monitor_ch: Sender<CompletedTest>,
441     concurrency: Concurrent,
442 ) {
443     let TestDescAndFn { desc, testfn } = test;
444
445     // Emscripten can catch panics but other wasm targets cannot
446     let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
447         && cfg!(target_arch = "wasm32") && !cfg!(target_os = "emscripten");
448
449     if force_ignore || desc.ignore || ignore_because_no_process_support {
450         let message = CompletedTest::new(desc, TrIgnored, None, Vec::new());
451         monitor_ch.send(message).unwrap();
452         return;
453     }
454
455     struct TestRunOpts {
456         pub strategy: RunStrategy,
457         pub nocapture: bool,
458         pub concurrency: Concurrent,
459         pub time: Option<time::TestTimeOptions>,
460     }
461
462     fn run_test_inner(
463         desc: TestDesc,
464         monitor_ch: Sender<CompletedTest>,
465         testfn: Box<dyn FnOnce() + Send>,
466         opts: TestRunOpts,
467     ) {
468         let concurrency = opts.concurrency;
469         let name = desc.name.clone();
470
471         let runtest = move || {
472             match opts.strategy {
473                 RunStrategy::InProcess =>
474                     run_test_in_process(
475                         desc,
476                         opts.nocapture,
477                         opts.time.is_some(),
478                         testfn,
479                         monitor_ch,
480                         opts.time
481                     ),
482                 RunStrategy::SpawnPrimary =>
483                     spawn_test_subprocess(desc, opts.time.is_some(), monitor_ch, opts.time),
484             }
485         };
486
487         // If the platform is single-threaded we're just going to run
488         // the test synchronously, regardless of the concurrency
489         // level.
490         let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
491         if concurrency == Concurrent::Yes && supports_threads {
492             let cfg = thread::Builder::new().name(name.as_slice().to_owned());
493             cfg.spawn(runtest).unwrap();
494         } else {
495             runtest();
496         }
497     }
498
499     let test_run_opts = TestRunOpts {
500         strategy,
501         nocapture: opts.nocapture,
502         concurrency,
503         time: opts.time_options
504     };
505
506     match testfn {
507         DynBenchFn(bencher) => {
508             // Benchmarks aren't expected to panic, so we run them all in-process.
509             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
510                 bencher.run(harness)
511             });
512         }
513         StaticBenchFn(benchfn) => {
514             // Benchmarks aren't expected to panic, so we run them all in-process.
515             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
516                 (benchfn.clone())(harness)
517             });
518         }
519         DynTestFn(f) => {
520             match strategy {
521                 RunStrategy::InProcess => (),
522                 _ => panic!("Cannot run dynamic test fn out-of-process"),
523             };
524             run_test_inner(
525                 desc,
526                 monitor_ch,
527                 Box::new(move || __rust_begin_short_backtrace(f)),
528                 test_run_opts,
529             );
530         }
531         StaticTestFn(f) => run_test_inner(
532             desc,
533             monitor_ch,
534             Box::new(move || __rust_begin_short_backtrace(f)),
535             test_run_opts,
536         ),
537     }
538 }
539
540 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
541 #[inline(never)]
542 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
543     f()
544 }
545
546 fn run_test_in_process(
547     desc: TestDesc,
548     nocapture: bool,
549     report_time: bool,
550     testfn: Box<dyn FnOnce() + Send>,
551     monitor_ch: Sender<CompletedTest>,
552     time_opts: Option<time::TestTimeOptions>,
553 ) {
554     // Buffer for capturing standard I/O
555     let data = Arc::new(Mutex::new(Vec::new()));
556
557     let oldio = if !nocapture {
558         Some((
559             io::set_print(Some(Sink::new_boxed(&data))),
560             io::set_panic(Some(Sink::new_boxed(&data))),
561         ))
562     } else {
563         None
564     };
565
566     let start = report_time.then(Instant::now);
567     let result = catch_unwind(AssertUnwindSafe(testfn));
568     let exec_time = start.map(|start| {
569         let duration = start.elapsed();
570         TestExecTime(duration)
571     });
572
573     if let Some((printio, panicio)) = oldio {
574         io::set_print(printio);
575         io::set_panic(panicio);
576     }
577
578     let test_result = match result {
579         Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
580         Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
581     };
582     let stdout = data.lock().unwrap().to_vec();
583     let message = CompletedTest::new(desc.clone(), test_result, exec_time, stdout);
584     monitor_ch.send(message).unwrap();
585 }
586
587 fn spawn_test_subprocess(
588     desc: TestDesc,
589     report_time: bool,
590     monitor_ch: Sender<CompletedTest>,
591     time_opts: Option<time::TestTimeOptions>,
592 ) {
593     let (result, test_output, exec_time) = (|| {
594         let args = env::args().collect::<Vec<_>>();
595         let current_exe = &args[0];
596
597         let start = report_time.then(Instant::now);
598         let output = match Command::new(current_exe)
599             .env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice())
600             .output() {
601                 Ok(out) => out,
602                 Err(e) => {
603                     let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
604                     return (TrFailed, err.into_bytes(), None);
605                 }
606             };
607         let exec_time = start.map(|start| {
608             let duration = start.elapsed();
609             TestExecTime(duration)
610         });
611
612         let std::process::Output { stdout, stderr, status } = output;
613         let mut test_output = stdout;
614         formatters::write_stderr_delimiter(&mut test_output, &desc.name);
615         test_output.extend_from_slice(&stderr);
616
617         let result = match (|| -> Result<TestResult, String> {
618             let exit_code = get_exit_code(status)?;
619             Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
620         })() {
621             Ok(r) => r,
622             Err(e) => {
623                 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
624                 TrFailed
625             }
626         };
627
628         (result, test_output, exec_time)
629     })();
630
631     let message = CompletedTest::new(desc.clone(), result, exec_time, test_output);
632     monitor_ch.send(message).unwrap();
633 }
634
635 fn run_test_in_spawned_subprocess(
636     desc: TestDesc,
637     testfn: Box<dyn FnOnce() + Send>,
638 ) -> ! {
639     let builtin_panic_hook = panic::take_hook();
640     let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
641         let test_result = match panic_info {
642             Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
643             None => calc_result(&desc, Ok(()), &None, &None),
644         };
645
646         // We don't support serializing TrFailedMsg, so just
647         // print the message out to stderr.
648         if let TrFailedMsg(msg) = &test_result {
649             eprintln!("{}", msg);
650         }
651
652         if let Some(info) = panic_info {
653             builtin_panic_hook(info);
654         }
655
656         if let TrOk = test_result {
657             process::exit(test_result::TR_OK);
658         } else {
659             process::exit(test_result::TR_FAILED);
660         }
661     });
662     let record_result2 = record_result.clone();
663     panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
664     testfn();
665     record_result(None);
666     unreachable!("panic=abort callback should have exited the process")
667 }