5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
53 options: Options::new(),
58 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
62 name: StaticTestName("1"),
64 #[cfg(not(bootstrap))]
66 should_panic: ShouldPanic::No,
69 test_type: TestType::Unknown,
73 testfn: DynTestFn(Box::new(move || {})),
77 name: StaticTestName("2"),
79 #[cfg(not(bootstrap))]
81 should_panic: ShouldPanic::No,
84 test_type: TestType::Unknown,
88 testfn: DynTestFn(Box::new(move || {})),
94 pub fn do_not_run_ignored_tests() {
98 let desc = TestDescAndFn {
100 name: StaticTestName("whatever"),
102 #[cfg(not(bootstrap))]
103 ignore_message: None,
104 should_panic: ShouldPanic::No,
107 test_type: TestType::Unknown,
111 testfn: DynTestFn(Box::new(f)),
113 let (tx, rx) = channel();
114 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
115 let result = rx.recv().unwrap().result;
116 assert_ne!(result, TrOk);
120 pub fn ignored_tests_result_in_ignored() {
122 let desc = TestDescAndFn {
124 name: StaticTestName("whatever"),
126 #[cfg(not(bootstrap))]
127 ignore_message: None,
128 should_panic: ShouldPanic::No,
131 test_type: TestType::Unknown,
135 testfn: DynTestFn(Box::new(f)),
137 let (tx, rx) = channel();
138 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
139 let result = rx.recv().unwrap().result;
140 assert_eq!(result, TrIgnored);
143 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
145 #[cfg(not(target_os = "emscripten"))]
146 fn test_should_panic() {
150 let desc = TestDescAndFn {
152 name: StaticTestName("whatever"),
154 #[cfg(not(bootstrap))]
155 ignore_message: None,
156 should_panic: ShouldPanic::Yes,
159 test_type: TestType::Unknown,
163 testfn: DynTestFn(Box::new(f)),
165 let (tx, rx) = channel();
166 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
167 let result = rx.recv().unwrap().result;
168 assert_eq!(result, TrOk);
171 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
173 #[cfg(not(target_os = "emscripten"))]
174 fn test_should_panic_good_message() {
176 panic!("an error message");
178 let desc = TestDescAndFn {
180 name: StaticTestName("whatever"),
182 #[cfg(not(bootstrap))]
183 ignore_message: None,
184 should_panic: ShouldPanic::YesWithMessage("error message"),
187 test_type: TestType::Unknown,
191 testfn: DynTestFn(Box::new(f)),
193 let (tx, rx) = channel();
194 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
195 let result = rx.recv().unwrap().result;
196 assert_eq!(result, TrOk);
199 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
201 #[cfg(not(target_os = "emscripten"))]
202 fn test_should_panic_bad_message() {
203 use crate::tests::TrFailedMsg;
205 panic!("an error message");
207 let expected = "foobar";
208 let failed_msg = r#"panic did not contain expected string
209 panic message: `"an error message"`,
210 expected substring: `"foobar"`"#;
211 let desc = TestDescAndFn {
213 name: StaticTestName("whatever"),
215 #[cfg(not(bootstrap))]
216 ignore_message: None,
217 should_panic: ShouldPanic::YesWithMessage(expected),
220 test_type: TestType::Unknown,
224 testfn: DynTestFn(Box::new(f)),
226 let (tx, rx) = channel();
227 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
228 let result = rx.recv().unwrap().result;
229 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
232 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
234 #[cfg(not(target_os = "emscripten"))]
235 fn test_should_panic_non_string_message_type() {
236 use crate::tests::TrFailedMsg;
237 use std::any::TypeId;
239 std::panic::panic_any(1i32);
241 let expected = "foobar";
242 let failed_msg = format!(
243 r#"expected panic with string value,
244 found non-string value: `{:?}`
245 expected substring: `"foobar"`"#,
248 let desc = TestDescAndFn {
250 name: StaticTestName("whatever"),
252 #[cfg(not(bootstrap))]
253 ignore_message: None,
254 should_panic: ShouldPanic::YesWithMessage(expected),
257 test_type: TestType::Unknown,
261 testfn: DynTestFn(Box::new(f)),
263 let (tx, rx) = channel();
264 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
265 let result = rx.recv().unwrap().result;
266 assert_eq!(result, TrFailedMsg(failed_msg));
269 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
271 #[cfg(not(target_os = "emscripten"))]
272 fn test_should_panic_but_succeeds() {
273 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
275 for &should_panic in should_panic_variants.iter() {
277 let desc = TestDescAndFn {
279 name: StaticTestName("whatever"),
281 #[cfg(not(bootstrap))]
282 ignore_message: None,
286 test_type: TestType::Unknown,
290 testfn: DynTestFn(Box::new(f)),
292 let (tx, rx) = channel();
298 RunStrategy::InProcess,
302 let result = rx.recv().unwrap().result;
305 TrFailedMsg("test did not panic as expected".to_string()),
306 "should_panic == {:?}",
312 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
314 let desc = TestDescAndFn {
316 name: StaticTestName("whatever"),
318 #[cfg(not(bootstrap))]
319 ignore_message: None,
320 should_panic: ShouldPanic::No,
323 test_type: TestType::Unknown,
327 testfn: DynTestFn(Box::new(f)),
329 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
331 let test_opts = TestOpts { time_options, ..TestOpts::new() };
332 let (tx, rx) = channel();
333 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
334 let exec_time = rx.recv().unwrap().exec_time;
339 fn test_should_not_report_time() {
340 let exec_time = report_time_test_template(false);
341 assert!(exec_time.is_none());
345 fn test_should_report_time() {
346 let exec_time = report_time_test_template(true);
347 assert!(exec_time.is_some());
350 fn time_test_failure_template(test_type: TestType) -> TestResult {
352 let desc = TestDescAndFn {
354 name: StaticTestName("whatever"),
356 #[cfg(not(bootstrap))]
357 ignore_message: None,
358 should_panic: ShouldPanic::No,
365 testfn: DynTestFn(Box::new(f)),
367 // `Default` will initialize all the thresholds to 0 milliseconds.
368 let mut time_options = TestTimeOptions::default();
369 time_options.error_on_excess = true;
371 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
372 let (tx, rx) = channel();
373 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
374 let result = rx.recv().unwrap().result;
380 fn test_error_on_exceed() {
381 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
383 for test_type in types.iter() {
384 let result = time_test_failure_template(*test_type);
386 assert_eq!(result, TestResult::TrTimedFail);
389 // Check that for unknown tests thresholds aren't applied.
390 let result = time_test_failure_template(TestType::Unknown);
391 assert_eq!(result, TestResult::TrOk);
394 fn typed_test_desc(test_type: TestType) -> TestDesc {
396 name: StaticTestName("whatever"),
398 #[cfg(not(bootstrap))]
399 ignore_message: None,
400 should_panic: ShouldPanic::No,
409 fn test_exec_time(millis: u64) -> TestExecTime {
410 TestExecTime(Duration::from_millis(millis))
414 fn test_time_options_threshold() {
415 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
416 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
417 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
419 let options = TestTimeOptions {
420 error_on_excess: false,
421 unit_threshold: unit.clone(),
422 integration_threshold: integration.clone(),
423 doctest_threshold: doc.clone(),
427 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
428 (TestType::UnitTest, unit.warn.as_millis(), true, false),
429 (TestType::UnitTest, unit.critical.as_millis(), true, true),
430 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
431 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
432 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
433 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
434 (TestType::DocTest, doc.warn.as_millis(), true, false),
435 (TestType::DocTest, doc.critical.as_millis(), true, true),
438 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
439 let test_desc = typed_test_desc(*test_type);
440 let exec_time = test_exec_time(*time as u64);
442 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
443 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
448 fn parse_ignored_flag() {
449 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
450 let opts = parse_opts(&args).unwrap().unwrap();
451 assert_eq!(opts.run_ignored, RunIgnored::Only);
455 fn parse_show_output_flag() {
456 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
457 let opts = parse_opts(&args).unwrap().unwrap();
458 assert!(opts.options.display_output);
462 fn parse_include_ignored_flag() {
463 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
464 let opts = parse_opts(&args).unwrap().unwrap();
465 assert_eq!(opts.run_ignored, RunIgnored::Yes);
469 pub fn filter_for_ignored_option() {
470 // When we run ignored tests the test filter should filter out all the
471 // unignored tests and flip the ignore flag on the rest to false
473 let mut opts = TestOpts::new();
474 opts.run_tests = true;
475 opts.run_ignored = RunIgnored::Only;
477 let tests = one_ignored_one_unignored_test();
478 let filtered = filter_tests(&opts, tests);
480 assert_eq!(filtered.len(), 1);
481 assert_eq!(filtered[0].desc.name.to_string(), "1");
482 assert!(!filtered[0].desc.ignore);
486 pub fn run_include_ignored_option() {
487 // When we "--include-ignored" tests, the ignore flag should be set to false on
488 // all tests and no test filtered out
490 let mut opts = TestOpts::new();
491 opts.run_tests = true;
492 opts.run_ignored = RunIgnored::Yes;
494 let tests = one_ignored_one_unignored_test();
495 let filtered = filter_tests(&opts, tests);
497 assert_eq!(filtered.len(), 2);
498 assert!(!filtered[0].desc.ignore);
499 assert!(!filtered[1].desc.ignore);
503 pub fn exclude_should_panic_option() {
504 let mut opts = TestOpts::new();
505 opts.run_tests = true;
506 opts.exclude_should_panic = true;
508 let mut tests = one_ignored_one_unignored_test();
509 tests.push(TestDescAndFn {
511 name: StaticTestName("3"),
513 #[cfg(not(bootstrap))]
514 ignore_message: None,
515 should_panic: ShouldPanic::Yes,
518 test_type: TestType::Unknown,
522 testfn: DynTestFn(Box::new(move || {})),
525 let filtered = filter_tests(&opts, tests);
527 assert_eq!(filtered.len(), 2);
528 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
532 pub fn exact_filter_match() {
533 fn tests() -> Vec<TestDescAndFn> {
534 ["base", "base::test", "base::test1", "base::test2"]
536 .map(|name| TestDescAndFn {
538 name: StaticTestName(name),
540 #[cfg(not(bootstrap))]
541 ignore_message: None,
542 should_panic: ShouldPanic::No,
545 test_type: TestType::Unknown,
549 testfn: DynTestFn(Box::new(move || {})),
555 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
556 assert_eq!(substr.len(), 4);
559 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
560 assert_eq!(substr.len(), 4);
563 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
564 assert_eq!(substr.len(), 3);
567 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
568 assert_eq!(substr.len(), 3);
570 let substr = filter_tests(
571 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
574 assert_eq!(substr.len(), 2);
576 let exact = filter_tests(
577 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
580 assert_eq!(exact.len(), 1);
582 let exact = filter_tests(
583 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
586 assert_eq!(exact.len(), 0);
588 let exact = filter_tests(
589 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
592 assert_eq!(exact.len(), 0);
594 let exact = filter_tests(
595 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
598 assert_eq!(exact.len(), 1);
600 let exact = filter_tests(
602 filters: vec!["base".into(), "base::test".into()],
608 assert_eq!(exact.len(), 2);
611 fn sample_tests() -> Vec<TestDescAndFn> {
613 "sha1::test".to_string(),
614 "isize::test_to_str".to_string(),
615 "isize::test_pow".to_string(),
616 "test::do_not_run_ignored_tests".to_string(),
617 "test::ignored_tests_result_in_ignored".to_string(),
618 "test::first_free_arg_should_be_a_filter".to_string(),
619 "test::parse_ignored_flag".to_string(),
620 "test::parse_include_ignored_flag".to_string(),
621 "test::filter_for_ignored_option".to_string(),
622 "test::run_include_ignored_option".to_string(),
623 "test::sort_tests".to_string(),
626 let mut tests = Vec::new();
628 let test = TestDescAndFn {
630 name: DynTestName((*name).clone()),
632 #[cfg(not(bootstrap))]
633 ignore_message: None,
634 should_panic: ShouldPanic::No,
637 test_type: TestType::Unknown,
641 testfn: DynTestFn(Box::new(testfn)),
649 pub fn sort_tests() {
650 let mut opts = TestOpts::new();
651 opts.run_tests = true;
653 let tests = sample_tests();
654 let filtered = filter_tests(&opts, tests);
657 "isize::test_pow".to_string(),
658 "isize::test_to_str".to_string(),
659 "sha1::test".to_string(),
660 "test::do_not_run_ignored_tests".to_string(),
661 "test::filter_for_ignored_option".to_string(),
662 "test::first_free_arg_should_be_a_filter".to_string(),
663 "test::ignored_tests_result_in_ignored".to_string(),
664 "test::parse_ignored_flag".to_string(),
665 "test::parse_include_ignored_flag".to_string(),
666 "test::run_include_ignored_option".to_string(),
667 "test::sort_tests".to_string(),
670 for (a, b) in expected.iter().zip(filtered) {
671 assert_eq!(*a, b.desc.name.to_string());
676 pub fn shuffle_tests() {
677 let mut opts = TestOpts::new();
680 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
683 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
685 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
687 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
689 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
691 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
695 pub fn shuffle_tests_with_seed() {
696 let mut opts = TestOpts::new();
699 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
702 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
704 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
706 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
707 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
709 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
713 pub fn order_depends_on_more_than_seed() {
714 let mut opts = TestOpts::new();
717 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
719 let mut left_tests = sample_tests();
720 let mut right_tests = sample_tests();
723 right_tests.remove(0);
726 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
728 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
730 assert_eq!(left.len(), right.len());
732 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
734 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
735 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
737 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
741 pub fn test_metricmap_compare() {
742 let mut m1 = MetricMap::new();
743 let mut m2 = MetricMap::new();
744 m1.insert_metric("in-both-noise", 1000.0, 200.0);
745 m2.insert_metric("in-both-noise", 1100.0, 200.0);
747 m1.insert_metric("in-first-noise", 1000.0, 2.0);
748 m2.insert_metric("in-second-noise", 1000.0, 2.0);
750 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
751 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
753 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
754 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
756 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
757 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
759 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
760 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
764 pub fn test_bench_once_no_iter() {
765 fn f(_: &mut Bencher) {}
770 pub fn test_bench_once_iter() {
771 fn f(b: &mut Bencher) {
778 pub fn test_bench_no_iter() {
779 fn f(_: &mut Bencher) {}
781 let (tx, rx) = channel();
783 let desc = TestDesc {
784 name: StaticTestName("f"),
786 #[cfg(not(bootstrap))]
787 ignore_message: None,
788 should_panic: ShouldPanic::No,
791 test_type: TestType::Unknown,
796 crate::bench::benchmark(TestId(0), desc, tx, true, f);
801 pub fn test_bench_iter() {
802 fn f(b: &mut Bencher) {
806 let (tx, rx) = channel();
808 let desc = TestDesc {
809 name: StaticTestName("f"),
811 #[cfg(not(bootstrap))]
812 ignore_message: None,
813 should_panic: ShouldPanic::No,
816 test_type: TestType::Unknown,
821 crate::bench::benchmark(TestId(0), desc, tx, true, f);
826 fn should_sort_failures_before_printing_them() {
827 let test_a = TestDesc {
828 name: StaticTestName("a"),
830 #[cfg(not(bootstrap))]
831 ignore_message: None,
832 should_panic: ShouldPanic::No,
835 test_type: TestType::Unknown,
840 let test_b = TestDesc {
841 name: StaticTestName("b"),
843 #[cfg(not(bootstrap))]
844 ignore_message: None,
845 should_panic: ShouldPanic::No,
848 test_type: TestType::Unknown,
853 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
855 let st = console::ConsoleTestState {
864 metrics: MetricMap::new(),
865 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
866 options: Options::new(),
867 not_failures: Vec::new(),
868 time_failures: Vec::new(),
871 out.write_failures(&st).unwrap();
872 let s = match out.output_location() {
873 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
874 &OutputLocation::Pretty(_) => unreachable!(),
877 let apos = s.find("a").unwrap();
878 let bpos = s.find("b").unwrap();
879 assert!(apos < bpos);