5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
53 options: Options::new(),
58 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
62 name: StaticTestName("1"),
65 should_panic: ShouldPanic::No,
68 test_type: TestType::Unknown,
70 testfn: DynTestFn(Box::new(move || Ok(()))),
74 name: StaticTestName("2"),
77 should_panic: ShouldPanic::No,
80 test_type: TestType::Unknown,
82 testfn: DynTestFn(Box::new(move || Ok(()))),
88 pub fn do_not_run_ignored_tests() {
89 fn f() -> Result<(), String> {
92 let desc = TestDescAndFn {
94 name: StaticTestName("whatever"),
97 should_panic: ShouldPanic::No,
100 test_type: TestType::Unknown,
102 testfn: DynTestFn(Box::new(f)),
104 let (tx, rx) = channel();
105 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
106 let result = rx.recv().unwrap().result;
107 assert_ne!(result, TrOk);
111 pub fn ignored_tests_result_in_ignored() {
112 fn f() -> Result<(), String> {
115 let desc = TestDescAndFn {
117 name: StaticTestName("whatever"),
119 ignore_message: None,
120 should_panic: ShouldPanic::No,
123 test_type: TestType::Unknown,
125 testfn: DynTestFn(Box::new(f)),
127 let (tx, rx) = channel();
128 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
129 let result = rx.recv().unwrap().result;
130 assert_eq!(result, TrIgnored);
133 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
135 #[cfg(not(target_os = "emscripten"))]
136 fn test_should_panic() {
137 fn f() -> Result<(), String> {
140 let desc = TestDescAndFn {
142 name: StaticTestName("whatever"),
144 ignore_message: None,
145 should_panic: ShouldPanic::Yes,
148 test_type: TestType::Unknown,
150 testfn: DynTestFn(Box::new(f)),
152 let (tx, rx) = channel();
153 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
154 let result = rx.recv().unwrap().result;
155 assert_eq!(result, TrOk);
158 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
160 #[cfg(not(target_os = "emscripten"))]
161 fn test_should_panic_good_message() {
162 fn f() -> Result<(), String> {
163 panic!("an error message");
165 let desc = TestDescAndFn {
167 name: StaticTestName("whatever"),
169 ignore_message: None,
170 should_panic: ShouldPanic::YesWithMessage("error message"),
173 test_type: TestType::Unknown,
175 testfn: DynTestFn(Box::new(f)),
177 let (tx, rx) = channel();
178 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
179 let result = rx.recv().unwrap().result;
180 assert_eq!(result, TrOk);
183 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
185 #[cfg(not(target_os = "emscripten"))]
186 fn test_should_panic_bad_message() {
187 use crate::tests::TrFailedMsg;
188 fn f() -> Result<(), String> {
189 panic!("an error message");
191 let expected = "foobar";
192 let failed_msg = r#"panic did not contain expected string
193 panic message: `"an error message"`,
194 expected substring: `"foobar"`"#;
195 let desc = TestDescAndFn {
197 name: StaticTestName("whatever"),
199 ignore_message: None,
200 should_panic: ShouldPanic::YesWithMessage(expected),
203 test_type: TestType::Unknown,
205 testfn: DynTestFn(Box::new(f)),
207 let (tx, rx) = channel();
208 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
209 let result = rx.recv().unwrap().result;
210 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
213 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
215 #[cfg(not(target_os = "emscripten"))]
216 fn test_should_panic_non_string_message_type() {
217 use crate::tests::TrFailedMsg;
218 use std::any::TypeId;
219 fn f() -> Result<(), String> {
220 std::panic::panic_any(1i32);
222 let expected = "foobar";
223 let failed_msg = format!(
224 r#"expected panic with string value,
225 found non-string value: `{:?}`
226 expected substring: `"foobar"`"#,
229 let desc = TestDescAndFn {
231 name: StaticTestName("whatever"),
233 ignore_message: None,
234 should_panic: ShouldPanic::YesWithMessage(expected),
237 test_type: TestType::Unknown,
239 testfn: DynTestFn(Box::new(f)),
241 let (tx, rx) = channel();
242 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
243 let result = rx.recv().unwrap().result;
244 assert_eq!(result, TrFailedMsg(failed_msg));
247 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
249 #[cfg(not(target_os = "emscripten"))]
250 fn test_should_panic_but_succeeds() {
251 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
253 for &should_panic in should_panic_variants.iter() {
254 fn f() -> Result<(), String> {
257 let desc = TestDescAndFn {
259 name: StaticTestName("whatever"),
261 ignore_message: None,
265 test_type: TestType::Unknown,
267 testfn: DynTestFn(Box::new(f)),
269 let (tx, rx) = channel();
275 RunStrategy::InProcess,
279 let result = rx.recv().unwrap().result;
282 TrFailedMsg("test did not panic as expected".to_string()),
283 "should_panic == {:?}",
289 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
290 fn f() -> Result<(), String> {
293 let desc = TestDescAndFn {
295 name: StaticTestName("whatever"),
297 ignore_message: None,
298 should_panic: ShouldPanic::No,
301 test_type: TestType::Unknown,
303 testfn: DynTestFn(Box::new(f)),
305 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
307 let test_opts = TestOpts { time_options, ..TestOpts::new() };
308 let (tx, rx) = channel();
309 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
310 let exec_time = rx.recv().unwrap().exec_time;
315 fn test_should_not_report_time() {
316 let exec_time = report_time_test_template(false);
317 assert!(exec_time.is_none());
321 fn test_should_report_time() {
322 let exec_time = report_time_test_template(true);
323 assert!(exec_time.is_some());
326 fn time_test_failure_template(test_type: TestType) -> TestResult {
327 fn f() -> Result<(), String> {
330 let desc = TestDescAndFn {
332 name: StaticTestName("whatever"),
334 ignore_message: None,
335 should_panic: ShouldPanic::No,
340 testfn: DynTestFn(Box::new(f)),
342 // `Default` will initialize all the thresholds to 0 milliseconds.
343 let mut time_options = TestTimeOptions::default();
344 time_options.error_on_excess = true;
346 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
347 let (tx, rx) = channel();
348 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
349 let result = rx.recv().unwrap().result;
355 fn test_error_on_exceed() {
356 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
358 for test_type in types.iter() {
359 let result = time_test_failure_template(*test_type);
361 assert_eq!(result, TestResult::TrTimedFail);
364 // Check that for unknown tests thresholds aren't applied.
365 let result = time_test_failure_template(TestType::Unknown);
366 assert_eq!(result, TestResult::TrOk);
369 fn typed_test_desc(test_type: TestType) -> TestDesc {
371 name: StaticTestName("whatever"),
373 ignore_message: None,
374 should_panic: ShouldPanic::No,
381 fn test_exec_time(millis: u64) -> TestExecTime {
382 TestExecTime(Duration::from_millis(millis))
386 fn test_time_options_threshold() {
387 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
388 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
389 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
391 let options = TestTimeOptions {
392 error_on_excess: false,
393 unit_threshold: unit.clone(),
394 integration_threshold: integration.clone(),
395 doctest_threshold: doc.clone(),
399 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
400 (TestType::UnitTest, unit.warn.as_millis(), true, false),
401 (TestType::UnitTest, unit.critical.as_millis(), true, true),
402 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
403 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
404 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
405 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
406 (TestType::DocTest, doc.warn.as_millis(), true, false),
407 (TestType::DocTest, doc.critical.as_millis(), true, true),
410 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
411 let test_desc = typed_test_desc(*test_type);
412 let exec_time = test_exec_time(*time as u64);
414 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
415 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
420 fn parse_ignored_flag() {
421 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
422 let opts = parse_opts(&args).unwrap().unwrap();
423 assert_eq!(opts.run_ignored, RunIgnored::Only);
427 fn parse_show_output_flag() {
428 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
429 let opts = parse_opts(&args).unwrap().unwrap();
430 assert!(opts.options.display_output);
434 fn parse_include_ignored_flag() {
435 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
436 let opts = parse_opts(&args).unwrap().unwrap();
437 assert_eq!(opts.run_ignored, RunIgnored::Yes);
441 pub fn filter_for_ignored_option() {
442 // When we run ignored tests the test filter should filter out all the
443 // unignored tests and flip the ignore flag on the rest to false
445 let mut opts = TestOpts::new();
446 opts.run_tests = true;
447 opts.run_ignored = RunIgnored::Only;
449 let tests = one_ignored_one_unignored_test();
450 let filtered = filter_tests(&opts, tests);
452 assert_eq!(filtered.len(), 1);
453 assert_eq!(filtered[0].desc.name.to_string(), "1");
454 assert!(!filtered[0].desc.ignore);
458 pub fn run_include_ignored_option() {
459 // When we "--include-ignored" tests, the ignore flag should be set to false on
460 // all tests and no test filtered out
462 let mut opts = TestOpts::new();
463 opts.run_tests = true;
464 opts.run_ignored = RunIgnored::Yes;
466 let tests = one_ignored_one_unignored_test();
467 let filtered = filter_tests(&opts, tests);
469 assert_eq!(filtered.len(), 2);
470 assert!(!filtered[0].desc.ignore);
471 assert!(!filtered[1].desc.ignore);
475 pub fn exclude_should_panic_option() {
476 let mut opts = TestOpts::new();
477 opts.run_tests = true;
478 opts.exclude_should_panic = true;
480 let mut tests = one_ignored_one_unignored_test();
481 tests.push(TestDescAndFn {
483 name: StaticTestName("3"),
485 ignore_message: None,
486 should_panic: ShouldPanic::Yes,
489 test_type: TestType::Unknown,
491 testfn: DynTestFn(Box::new(move || Ok(()))),
494 let filtered = filter_tests(&opts, tests);
496 assert_eq!(filtered.len(), 2);
497 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
501 pub fn exact_filter_match() {
502 fn tests() -> Vec<TestDescAndFn> {
503 ["base", "base::test", "base::test1", "base::test2"]
505 .map(|name| TestDescAndFn {
507 name: StaticTestName(name),
509 ignore_message: None,
510 should_panic: ShouldPanic::No,
513 test_type: TestType::Unknown,
515 testfn: DynTestFn(Box::new(move || Ok(()))),
521 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
522 assert_eq!(substr.len(), 4);
525 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
526 assert_eq!(substr.len(), 4);
529 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
530 assert_eq!(substr.len(), 3);
533 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
534 assert_eq!(substr.len(), 3);
536 let substr = filter_tests(
537 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
540 assert_eq!(substr.len(), 2);
542 let exact = filter_tests(
543 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
546 assert_eq!(exact.len(), 1);
548 let exact = filter_tests(
549 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
552 assert_eq!(exact.len(), 0);
554 let exact = filter_tests(
555 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
558 assert_eq!(exact.len(), 0);
560 let exact = filter_tests(
561 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
564 assert_eq!(exact.len(), 1);
566 let exact = filter_tests(
568 filters: vec!["base".into(), "base::test".into()],
574 assert_eq!(exact.len(), 2);
577 fn sample_tests() -> Vec<TestDescAndFn> {
579 "sha1::test".to_string(),
580 "isize::test_to_str".to_string(),
581 "isize::test_pow".to_string(),
582 "test::do_not_run_ignored_tests".to_string(),
583 "test::ignored_tests_result_in_ignored".to_string(),
584 "test::first_free_arg_should_be_a_filter".to_string(),
585 "test::parse_ignored_flag".to_string(),
586 "test::parse_include_ignored_flag".to_string(),
587 "test::filter_for_ignored_option".to_string(),
588 "test::run_include_ignored_option".to_string(),
589 "test::sort_tests".to_string(),
591 fn testfn() -> Result<(), String> {
594 let mut tests = Vec::new();
596 let test = TestDescAndFn {
598 name: DynTestName((*name).clone()),
600 ignore_message: None,
601 should_panic: ShouldPanic::No,
604 test_type: TestType::Unknown,
606 testfn: DynTestFn(Box::new(testfn)),
614 pub fn sort_tests() {
615 let mut opts = TestOpts::new();
616 opts.run_tests = true;
618 let tests = sample_tests();
619 let filtered = filter_tests(&opts, tests);
622 "isize::test_pow".to_string(),
623 "isize::test_to_str".to_string(),
624 "sha1::test".to_string(),
625 "test::do_not_run_ignored_tests".to_string(),
626 "test::filter_for_ignored_option".to_string(),
627 "test::first_free_arg_should_be_a_filter".to_string(),
628 "test::ignored_tests_result_in_ignored".to_string(),
629 "test::parse_ignored_flag".to_string(),
630 "test::parse_include_ignored_flag".to_string(),
631 "test::run_include_ignored_option".to_string(),
632 "test::sort_tests".to_string(),
635 for (a, b) in expected.iter().zip(filtered) {
636 assert_eq!(*a, b.desc.name.to_string());
641 pub fn shuffle_tests() {
642 let mut opts = TestOpts::new();
645 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
648 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
650 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
652 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
654 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
656 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
660 pub fn shuffle_tests_with_seed() {
661 let mut opts = TestOpts::new();
664 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
667 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
669 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
671 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
672 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
674 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
678 pub fn order_depends_on_more_than_seed() {
679 let mut opts = TestOpts::new();
682 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
684 let mut left_tests = sample_tests();
685 let mut right_tests = sample_tests();
688 right_tests.remove(0);
691 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
693 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
695 assert_eq!(left.len(), right.len());
697 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
699 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
700 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
702 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
706 pub fn test_metricmap_compare() {
707 let mut m1 = MetricMap::new();
708 let mut m2 = MetricMap::new();
709 m1.insert_metric("in-both-noise", 1000.0, 200.0);
710 m2.insert_metric("in-both-noise", 1100.0, 200.0);
712 m1.insert_metric("in-first-noise", 1000.0, 2.0);
713 m2.insert_metric("in-second-noise", 1000.0, 2.0);
715 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
716 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
718 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
719 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
721 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
722 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
724 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
725 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
729 pub fn test_bench_once_no_iter() {
730 fn f(_: &mut Bencher) -> Result<(), String> {
733 bench::run_once(f).unwrap();
737 pub fn test_bench_once_iter() {
738 fn f(b: &mut Bencher) -> Result<(), String> {
742 bench::run_once(f).unwrap();
746 pub fn test_bench_no_iter() {
747 fn f(_: &mut Bencher) -> Result<(), String> {
751 let (tx, rx) = channel();
753 let desc = TestDesc {
754 name: StaticTestName("f"),
756 ignore_message: None,
757 should_panic: ShouldPanic::No,
760 test_type: TestType::Unknown,
763 crate::bench::benchmark(TestId(0), desc, tx, true, f);
768 pub fn test_bench_iter() {
769 fn f(b: &mut Bencher) -> Result<(), String> {
774 let (tx, rx) = channel();
776 let desc = TestDesc {
777 name: StaticTestName("f"),
779 ignore_message: None,
780 should_panic: ShouldPanic::No,
783 test_type: TestType::Unknown,
786 crate::bench::benchmark(TestId(0), desc, tx, true, f);
791 fn should_sort_failures_before_printing_them() {
792 let test_a = TestDesc {
793 name: StaticTestName("a"),
795 ignore_message: None,
796 should_panic: ShouldPanic::No,
799 test_type: TestType::Unknown,
802 let test_b = TestDesc {
803 name: StaticTestName("b"),
805 ignore_message: None,
806 should_panic: ShouldPanic::No,
809 test_type: TestType::Unknown,
812 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
814 let st = console::ConsoleTestState {
823 metrics: MetricMap::new(),
824 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
825 options: Options::new(),
826 not_failures: Vec::new(),
827 time_failures: Vec::new(),
830 out.write_failures(&st).unwrap();
831 let s = match out.output_location() {
832 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
833 &OutputLocation::Pretty(_) => unreachable!(),
836 let apos = s.find("a").unwrap();
837 let bpos = s.find("b").unwrap();
838 assert!(apos < bpos);
842 #[cfg(not(target_os = "emscripten"))]
843 fn test_dyn_bench_returning_err_fails_when_run_as_test() {
844 fn f(_: &mut Bencher) -> Result<(), String> {
845 Result::Err("An error".into())
847 let desc = TestDescAndFn {
849 name: StaticTestName("whatever"),
851 ignore_message: None,
852 should_panic: ShouldPanic::No,
855 test_type: TestType::Unknown,
857 testfn: DynBenchFn(Box::new(f)),
859 let (tx, rx) = channel();
860 let notify = move |event: TestEvent| {
861 if let TestEvent::TeResult(result) = event {
862 tx.send(result).unwrap();
866 run_tests(&TestOpts { run_tests: true, ..TestOpts::new() }, vec![desc], notify).unwrap();
867 let result = rx.recv().unwrap().result;
868 assert_eq!(result, TrFailed);