]> git.lizzy.rs Git - rust.git/blob - src/libtest/lib.rs
Update Clippy
[rust.git] / src / libtest / lib.rs
1 //! Support code for rustc's built in unit-test and micro-benchmarking
2 //! framework.
3 //!
4 //! Almost all user code will only be interested in `Bencher` and
5 //! `black_box`. All other interactions (such as writing tests and
6 //! benchmarks themselves) should be done via the `#[test]` and
7 //! `#[bench]` attributes.
8 //!
9 //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
10
11 // Currently, not much of this is meant for users. It is intended to
12 // support the simplest interface possible for representing and
13 // running tests while providing a base that other test frameworks may
14 // build off of.
15
16 // N.B., this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
17 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18 // cargo) to detect this crate.
19
20 #![crate_name = "test"]
21 #![unstable(feature = "test", issue = "50297")]
22 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
23 #![feature(asm)]
24 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
25 #![feature(rustc_private)]
26 #![feature(nll)]
27 #![feature(set_stdio)]
28 #![feature(panic_unwind)]
29 #![feature(staged_api)]
30 #![feature(termination_trait_lib)]
31 #![feature(test)]
32
33 // Public reexports
34 pub use self::ColorConfig::*;
35 pub use self::types::*;
36 pub use self::types::TestName::*;
37 pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
38 pub use self::bench::{Bencher, black_box};
39 pub use self::console::run_tests_console;
40 pub use cli::TestOpts;
41
42 // Module to be used by rustc to compile tests in libtest
43 pub mod test {
44     pub use crate::{
45         bench::Bencher,
46         cli::{parse_opts, TestOpts},
47         helpers::metrics::{Metric, MetricMap},
48         options::{ShouldPanic, Options, RunIgnored, RunStrategy},
49         test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
50         time::{TestTimeOptions, TestExecTime},
51         types::{
52             DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName,
53             TestDesc, TestDescAndFn, TestName, TestType,
54         },
55         assert_test_result, filter_tests, run_test, test_main, test_main_static,
56     };
57 }
58
59 use std::{
60     env,
61     io,
62     io::prelude::Write,
63     panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
64     process,
65     process::{Command, Termination},
66     sync::mpsc::{channel, Sender},
67     sync::{Arc, Mutex},
68     thread,
69     time::{Duration, Instant},
70 };
71
72 pub mod stats;
73 pub mod bench;
74 mod formatters;
75 mod cli;
76 mod console;
77 mod event;
78 mod helpers;
79 mod time;
80 mod types;
81 mod options;
82 mod test_result;
83
84 #[cfg(test)]
85 mod tests;
86
87 use test_result::*;
88 use time::TestExecTime;
89 use options::{RunStrategy, Concurrent};
90 use event::{CompletedTest, TestEvent};
91 use helpers::sink::Sink;
92 use helpers::concurrency::get_concurrency;
93 use helpers::exit_code::get_exit_code;
94
95 // Process exit code to be used to indicate test failures.
96 const ERROR_EXIT_CODE: i32 = 101;
97
98 const SECONDARY_TEST_INVOKER_VAR: &'static str = "__RUST_TEST_INVOKE";
99
100 // The default console test runner. It accepts the command line
101 // arguments and a vector of test_descs.
102 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
103     let mut opts = match cli::parse_opts(args) {
104         Some(Ok(o)) => o,
105         Some(Err(msg)) => {
106             eprintln!("error: {}", msg);
107             process::exit(ERROR_EXIT_CODE);
108         }
109         None => return,
110     };
111     if let Some(options) = options {
112         opts.options = options;
113     }
114     if opts.list {
115         if let Err(e) = console::list_tests_console(&opts, tests) {
116             eprintln!("error: io error when listing tests: {:?}", e);
117             process::exit(ERROR_EXIT_CODE);
118         }
119     } else {
120         match console::run_tests_console(&opts, tests) {
121             Ok(true) => {}
122             Ok(false) => process::exit(ERROR_EXIT_CODE),
123             Err(e) => {
124                 eprintln!("error: io error when listing tests: {:?}", e);
125                 process::exit(ERROR_EXIT_CODE);
126             }
127         }
128     }
129 }
130
131 /// A variant optimized for invocation with a static test vector.
132 /// This will panic (intentionally) when fed any dynamic tests.
133 ///
134 /// This is the entry point for the main function generated by `rustc --test`
135 /// when panic=unwind.
136 pub fn test_main_static(tests: &[&TestDescAndFn]) {
137     let args = env::args().collect::<Vec<_>>();
138     let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
139     test_main(&args, owned_tests, None)
140 }
141
142 /// A variant optimized for invocation with a static test vector.
143 /// This will panic (intentionally) when fed any dynamic tests.
144 ///
145 /// Runs tests in panic=abort mode, which involves spawning subprocesses for
146 /// tests.
147 ///
148 /// This is the entry point for the main function generated by `rustc --test`
149 /// when panic=abort.
150 pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
151     // If we're being run in SpawnedSecondary mode, run the test here. run_test
152     // will then exit the process.
153     if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
154         let test = tests
155             .iter()
156             .filter(|test| test.desc.name.as_slice() == name)
157             .map(make_owned_test)
158             .next()
159             .expect("couldn't find a test with the provided name");
160         let TestDescAndFn { desc, testfn } = test;
161         let testfn = match testfn {
162             StaticTestFn(f) => f,
163             _ => panic!("only static tests are supported"),
164         };
165         run_test_in_spawned_subprocess(desc, Box::new(testfn));
166     }
167
168     let args = env::args().collect::<Vec<_>>();
169     let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
170     test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
171 }
172
173 /// Clones static values for putting into a dynamic vector, which test_main()
174 /// needs to hand out ownership of tests to parallel test runners.
175 ///
176 /// This will panic when fed any dynamic tests, because they cannot be cloned.
177 fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
178     match test.testfn {
179         StaticTestFn(f) => TestDescAndFn {
180             testfn: StaticTestFn(f),
181             desc: test.desc.clone(),
182         },
183         StaticBenchFn(f) => TestDescAndFn {
184             testfn: StaticBenchFn(f),
185             desc: test.desc.clone(),
186         },
187         _ => panic!("non-static tests passed to test::test_main_static"),
188     }
189 }
190
191 /// Invoked when unit tests terminate. Should panic if the unit
192 /// Tests is considered a failure. By default, invokes `report()`
193 /// and checks for a `0` result.
194 pub fn assert_test_result<T: Termination>(result: T) {
195     let code = result.report();
196     assert_eq!(
197         code, 0,
198         "the test returned a termination value with a non-zero status code ({}) \
199          which indicates a failure",
200         code
201     );
202 }
203
204 pub fn run_tests<F>(
205     opts: &TestOpts,
206     tests: Vec<TestDescAndFn>,
207     mut notify_about_test_event: F
208 ) -> io::Result<()>
209 where
210     F: FnMut(TestEvent) -> io::Result<()>,
211 {
212     use std::collections::{self, HashMap};
213     use std::hash::BuildHasherDefault;
214     use std::sync::mpsc::RecvTimeoutError;
215     // Use a deterministic hasher
216     type TestMap =
217         HashMap<TestDesc, Instant, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
218
219     let tests_len = tests.len();
220
221     let mut filtered_tests = filter_tests(opts, tests);
222     if !opts.bench_benchmarks {
223         filtered_tests = convert_benchmarks_to_tests(filtered_tests);
224     }
225
226     let filtered_tests = {
227         let mut filtered_tests = filtered_tests;
228         for test in filtered_tests.iter_mut() {
229             test.desc.name = test.desc.name.with_padding(test.testfn.padding());
230         }
231
232         filtered_tests
233     };
234
235     let filtered_out = tests_len - filtered_tests.len();
236     let event = TestEvent::TeFilteredOut(filtered_out);
237     notify_about_test_event(event)?;
238
239     let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
240
241     let event = TestEvent::TeFiltered(filtered_descs);
242     notify_about_test_event(event)?;
243
244     let (filtered_tests, filtered_benchs): (Vec<_>, _) =
245         filtered_tests.into_iter().partition(|e| match e.testfn {
246             StaticTestFn(_) | DynTestFn(_) => true,
247             _ => false,
248         });
249
250     let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
251
252     let mut remaining = filtered_tests;
253     remaining.reverse();
254     let mut pending = 0;
255
256     let (tx, rx) = channel::<CompletedTest>();
257     let run_strategy = if opts.options.panic_abort {
258         RunStrategy::SpawnPrimary
259     } else {
260         RunStrategy::InProcess
261     };
262
263     let mut running_tests: TestMap = HashMap::default();
264
265     fn get_timed_out_tests(running_tests: &mut TestMap) -> Vec<TestDesc> {
266         let now = Instant::now();
267         let timed_out = running_tests
268             .iter()
269             .filter_map(|(desc, timeout)| {
270                 if &now >= timeout {
271                     Some(desc.clone())
272                 } else {
273                     None
274                 }
275             })
276             .collect();
277         for test in &timed_out {
278             running_tests.remove(test);
279         }
280         timed_out
281     };
282
283     fn calc_timeout(running_tests: &TestMap) -> Option<Duration> {
284         running_tests.values().min().map(|next_timeout| {
285             let now = Instant::now();
286             if *next_timeout >= now {
287                 *next_timeout - now
288             } else {
289                 Duration::new(0, 0)
290             }
291         })
292     };
293
294     if concurrency == 1 {
295         while !remaining.is_empty() {
296             let test = remaining.pop().unwrap();
297             let event = TestEvent::TeWait(test.desc.clone());
298             notify_about_test_event(event)?;
299             run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No);
300             let completed_test = rx.recv().unwrap();
301
302             let event = TestEvent::TeResult(completed_test);
303             notify_about_test_event(event)?;
304         }
305     } else {
306         while pending > 0 || !remaining.is_empty() {
307             while pending < concurrency && !remaining.is_empty() {
308                 let test = remaining.pop().unwrap();
309                 let timeout = time::get_default_test_timeout();
310                 running_tests.insert(test.desc.clone(), timeout);
311
312                 let event = TestEvent::TeWait(test.desc.clone());
313                 notify_about_test_event(event)?; //here no pad
314                 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::Yes);
315                 pending += 1;
316             }
317
318             let mut res;
319             loop {
320                 if let Some(timeout) = calc_timeout(&running_tests) {
321                     res = rx.recv_timeout(timeout);
322                     for test in get_timed_out_tests(&mut running_tests) {
323                         let event = TestEvent::TeTimeout(test);
324                         notify_about_test_event(event)?;
325                     }
326
327                     match res {
328                         Err(RecvTimeoutError::Timeout) => {
329                             // Result is not yet ready, continue waiting.
330                         }
331                         _ => {
332                             // We've got a result, stop the loop.
333                             break;
334                         }
335                     }
336                 } else {
337                     res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
338                     break;
339                 }
340             }
341
342             let completed_test = res.unwrap();
343             running_tests.remove(&completed_test.desc);
344
345             let event = TestEvent::TeResult(completed_test);
346             notify_about_test_event(event)?;
347             pending -= 1;
348         }
349     }
350
351     if opts.bench_benchmarks {
352         // All benchmarks run at the end, in serial.
353         for b in filtered_benchs {
354             let event = TestEvent::TeWait(b.desc.clone());
355             notify_about_test_event(event)?;
356             run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
357             let completed_test = rx.recv().unwrap();
358
359             let event = TestEvent::TeResult(completed_test);
360             notify_about_test_event(event)?;
361         }
362     }
363     Ok(())
364 }
365
366 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
367     let mut filtered = tests;
368     let matches_filter = |test: &TestDescAndFn, filter: &str| {
369         let test_name = test.desc.name.as_slice();
370
371         match opts.filter_exact {
372             true => test_name == filter,
373             false => test_name.contains(filter),
374         }
375     };
376
377     // Remove tests that don't match the test filter
378     if let Some(ref filter) = opts.filter {
379         filtered.retain(|test| matches_filter(test, filter));
380     }
381
382     // Skip tests that match any of the skip filters
383     filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
384
385     // Excludes #[should_panic] tests
386     if opts.exclude_should_panic {
387         filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
388     }
389
390     // maybe unignore tests
391     match opts.run_ignored {
392         RunIgnored::Yes => {
393             filtered
394                 .iter_mut()
395                 .for_each(|test| test.desc.ignore = false);
396         }
397         RunIgnored::Only => {
398             filtered.retain(|test| test.desc.ignore);
399             filtered
400                 .iter_mut()
401                 .for_each(|test| test.desc.ignore = false);
402         }
403         RunIgnored::No => {}
404     }
405
406     // Sort the tests alphabetically
407     filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
408
409     filtered
410 }
411
412 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
413     // convert benchmarks to tests, if we're not benchmarking them
414     tests
415         .into_iter()
416         .map(|x| {
417             let testfn = match x.testfn {
418                 DynBenchFn(bench) => DynTestFn(Box::new(move || {
419                     bench::run_once(|b| __rust_begin_short_backtrace(|| bench.run(b)))
420                 })),
421                 StaticBenchFn(benchfn) => DynTestFn(Box::new(move || {
422                     bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b)))
423                 })),
424                 f => f,
425             };
426             TestDescAndFn {
427                 desc: x.desc,
428                 testfn,
429             }
430         })
431         .collect()
432 }
433
434 pub fn run_test(
435     opts: &TestOpts,
436     force_ignore: bool,
437     test: TestDescAndFn,
438     strategy: RunStrategy,
439     monitor_ch: Sender<CompletedTest>,
440     concurrency: Concurrent,
441 ) {
442     let TestDescAndFn { desc, testfn } = test;
443
444     // FIXME: Re-enable emscripten once it can catch panics again
445     let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
446         && (cfg!(target_arch = "wasm32") || cfg!(target_os = "emscripten"));
447
448     if force_ignore || desc.ignore || ignore_because_no_process_support {
449         let message = CompletedTest::new(desc, TrIgnored, None, Vec::new());
450         monitor_ch.send(message).unwrap();
451         return;
452     }
453
454     struct TestRunOpts {
455         pub strategy: RunStrategy,
456         pub nocapture: bool,
457         pub concurrency: Concurrent,
458         pub time: Option<time::TestTimeOptions>,
459     }
460
461     fn run_test_inner(
462         desc: TestDesc,
463         monitor_ch: Sender<CompletedTest>,
464         testfn: Box<dyn FnOnce() + Send>,
465         opts: TestRunOpts,
466     ) {
467         let concurrency = opts.concurrency;
468         let name = desc.name.clone();
469
470         let runtest = move || {
471             match opts.strategy {
472                 RunStrategy::InProcess =>
473                     run_test_in_process(
474                         desc,
475                         opts.nocapture,
476                         opts.time.is_some(),
477                         testfn,
478                         monitor_ch,
479                         opts.time
480                     ),
481                 RunStrategy::SpawnPrimary =>
482                     spawn_test_subprocess(desc, opts.time.is_some(), monitor_ch, opts.time),
483             }
484         };
485
486         // If the platform is single-threaded we're just going to run
487         // the test synchronously, regardless of the concurrency
488         // level.
489         let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
490         if concurrency == Concurrent::Yes && supports_threads {
491             let cfg = thread::Builder::new().name(name.as_slice().to_owned());
492             cfg.spawn(runtest).unwrap();
493         } else {
494             runtest();
495         }
496     }
497
498     let test_run_opts = TestRunOpts {
499         strategy,
500         nocapture: opts.nocapture,
501         concurrency,
502         time: opts.time_options
503     };
504
505     match testfn {
506         DynBenchFn(bencher) => {
507             // Benchmarks aren't expected to panic, so we run them all in-process.
508             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
509                 bencher.run(harness)
510             });
511         }
512         StaticBenchFn(benchfn) => {
513             // Benchmarks aren't expected to panic, so we run them all in-process.
514             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
515                 (benchfn.clone())(harness)
516             });
517         }
518         DynTestFn(f) => {
519             match strategy {
520                 RunStrategy::InProcess => (),
521                 _ => panic!("Cannot run dynamic test fn out-of-process"),
522             };
523             run_test_inner(
524                 desc,
525                 monitor_ch,
526                 Box::new(move || __rust_begin_short_backtrace(f)),
527                 test_run_opts,
528             );
529         }
530         StaticTestFn(f) => run_test_inner(
531             desc,
532             monitor_ch,
533             Box::new(move || __rust_begin_short_backtrace(f)),
534             test_run_opts,
535         ),
536     }
537 }
538
539 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
540 #[inline(never)]
541 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
542     f()
543 }
544
545 fn run_test_in_process(
546     desc: TestDesc,
547     nocapture: bool,
548     report_time: bool,
549     testfn: Box<dyn FnOnce() + Send>,
550     monitor_ch: Sender<CompletedTest>,
551     time_opts: Option<time::TestTimeOptions>,
552 ) {
553     // Buffer for capturing standard I/O
554     let data = Arc::new(Mutex::new(Vec::new()));
555
556     let oldio = if !nocapture {
557         Some((
558             io::set_print(Some(Sink::new_boxed(&data))),
559             io::set_panic(Some(Sink::new_boxed(&data))),
560         ))
561     } else {
562         None
563     };
564
565     let start = if report_time {
566         Some(Instant::now())
567     } else {
568         None
569     };
570     let result = catch_unwind(AssertUnwindSafe(testfn));
571     let exec_time = start.map(|start| {
572         let duration = start.elapsed();
573         TestExecTime(duration)
574     });
575
576     if let Some((printio, panicio)) = oldio {
577         io::set_print(printio);
578         io::set_panic(panicio);
579     }
580
581     let test_result = match result {
582         Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
583         Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
584     };
585     let stdout = data.lock().unwrap().to_vec();
586     let message = CompletedTest::new(desc.clone(), test_result, exec_time, stdout);
587     monitor_ch.send(message).unwrap();
588 }
589
590 fn spawn_test_subprocess(
591     desc: TestDesc,
592     report_time: bool,
593     monitor_ch: Sender<CompletedTest>,
594     time_opts: Option<time::TestTimeOptions>,
595 ) {
596     let (result, test_output, exec_time) = (|| {
597         let args = env::args().collect::<Vec<_>>();
598         let current_exe = &args[0];
599
600         let start = if report_time {
601             Some(Instant::now())
602         } else {
603             None
604         };
605         let output = match Command::new(current_exe)
606             .env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice())
607             .output() {
608                 Ok(out) => out,
609                 Err(e) => {
610                     let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
611                     return (TrFailed, err.into_bytes(), None);
612                 }
613             };
614         let exec_time = start.map(|start| {
615             let duration = start.elapsed();
616             TestExecTime(duration)
617         });
618
619         let std::process::Output { stdout, stderr, status } = output;
620         let mut test_output = stdout;
621         formatters::write_stderr_delimiter(&mut test_output, &desc.name);
622         test_output.extend_from_slice(&stderr);
623
624         let result = match (|| -> Result<TestResult, String> {
625             let exit_code = get_exit_code(status)?;
626             Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
627         })() {
628             Ok(r) => r,
629             Err(e) => {
630                 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
631                 TrFailed
632             }
633         };
634
635         (result, test_output, exec_time)
636     })();
637
638     let message = CompletedTest::new(desc.clone(), result, exec_time, test_output);
639     monitor_ch.send(message).unwrap();
640 }
641
642 fn run_test_in_spawned_subprocess(
643     desc: TestDesc,
644     testfn: Box<dyn FnOnce() + Send>,
645 ) -> ! {
646     let builtin_panic_hook = panic::take_hook();
647     let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
648         let test_result = match panic_info {
649             Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
650             None => calc_result(&desc, Ok(()), &None, &None),
651         };
652
653         // We don't support serializing TrFailedMsg, so just
654         // print the message out to stderr.
655         if let TrFailedMsg(msg) = &test_result {
656             eprintln!("{}", msg);
657         }
658
659         if let Some(info) = panic_info {
660             builtin_panic_hook(info);
661         }
662
663         if let TrOk = test_result {
664             process::exit(test_result::TR_OK);
665         } else {
666             process::exit(test_result::TR_FAILED);
667         }
668     });
669     let record_result2 = record_result.clone();
670     panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
671     testfn();
672     record_result(None);
673     unreachable!("panic=abort callback should have exited the process")
674 }