5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
53 options: Options::new(),
58 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
62 name: StaticTestName("1"),
64 #[cfg(not(bootstrap))]
66 should_panic: ShouldPanic::No,
69 test_type: TestType::Unknown,
71 testfn: DynTestFn(Box::new(move || {})),
75 name: StaticTestName("2"),
77 #[cfg(not(bootstrap))]
79 should_panic: ShouldPanic::No,
82 test_type: TestType::Unknown,
84 testfn: DynTestFn(Box::new(move || {})),
90 pub fn do_not_run_ignored_tests() {
94 let desc = TestDescAndFn {
96 name: StaticTestName("whatever"),
98 #[cfg(not(bootstrap))]
100 should_panic: ShouldPanic::No,
103 test_type: TestType::Unknown,
105 testfn: DynTestFn(Box::new(f)),
107 let (tx, rx) = channel();
108 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
109 let result = rx.recv().unwrap().result;
110 assert_ne!(result, TrOk);
114 pub fn ignored_tests_result_in_ignored() {
116 let desc = TestDescAndFn {
118 name: StaticTestName("whatever"),
120 #[cfg(not(bootstrap))]
121 ignore_message: None,
122 should_panic: ShouldPanic::No,
125 test_type: TestType::Unknown,
127 testfn: DynTestFn(Box::new(f)),
129 let (tx, rx) = channel();
130 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
131 let result = rx.recv().unwrap().result;
132 assert_eq!(result, TrIgnored);
135 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
137 #[cfg(not(target_os = "emscripten"))]
138 fn test_should_panic() {
142 let desc = TestDescAndFn {
144 name: StaticTestName("whatever"),
146 #[cfg(not(bootstrap))]
147 ignore_message: None,
148 should_panic: ShouldPanic::Yes,
151 test_type: TestType::Unknown,
153 testfn: DynTestFn(Box::new(f)),
155 let (tx, rx) = channel();
156 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
157 let result = rx.recv().unwrap().result;
158 assert_eq!(result, TrOk);
161 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
163 #[cfg(not(target_os = "emscripten"))]
164 fn test_should_panic_good_message() {
166 panic!("an error message");
168 let desc = TestDescAndFn {
170 name: StaticTestName("whatever"),
172 #[cfg(not(bootstrap))]
173 ignore_message: None,
174 should_panic: ShouldPanic::YesWithMessage("error message"),
177 test_type: TestType::Unknown,
179 testfn: DynTestFn(Box::new(f)),
181 let (tx, rx) = channel();
182 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
183 let result = rx.recv().unwrap().result;
184 assert_eq!(result, TrOk);
187 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
189 #[cfg(not(target_os = "emscripten"))]
190 fn test_should_panic_bad_message() {
191 use crate::tests::TrFailedMsg;
193 panic!("an error message");
195 let expected = "foobar";
196 let failed_msg = r#"panic did not contain expected string
197 panic message: `"an error message"`,
198 expected substring: `"foobar"`"#;
199 let desc = TestDescAndFn {
201 name: StaticTestName("whatever"),
203 #[cfg(not(bootstrap))]
204 ignore_message: None,
205 should_panic: ShouldPanic::YesWithMessage(expected),
208 test_type: TestType::Unknown,
210 testfn: DynTestFn(Box::new(f)),
212 let (tx, rx) = channel();
213 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
214 let result = rx.recv().unwrap().result;
215 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
218 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
220 #[cfg(not(target_os = "emscripten"))]
221 fn test_should_panic_non_string_message_type() {
222 use crate::tests::TrFailedMsg;
223 use std::any::TypeId;
225 std::panic::panic_any(1i32);
227 let expected = "foobar";
228 let failed_msg = format!(
229 r#"expected panic with string value,
230 found non-string value: `{:?}`
231 expected substring: `"foobar"`"#,
234 let desc = TestDescAndFn {
236 name: StaticTestName("whatever"),
238 #[cfg(not(bootstrap))]
239 ignore_message: None,
240 should_panic: ShouldPanic::YesWithMessage(expected),
243 test_type: TestType::Unknown,
245 testfn: DynTestFn(Box::new(f)),
247 let (tx, rx) = channel();
248 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
249 let result = rx.recv().unwrap().result;
250 assert_eq!(result, TrFailedMsg(failed_msg));
253 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
255 #[cfg(not(target_os = "emscripten"))]
256 fn test_should_panic_but_succeeds() {
257 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
259 for &should_panic in should_panic_variants.iter() {
261 let desc = TestDescAndFn {
263 name: StaticTestName("whatever"),
265 #[cfg(not(bootstrap))]
266 ignore_message: None,
270 test_type: TestType::Unknown,
272 testfn: DynTestFn(Box::new(f)),
274 let (tx, rx) = channel();
280 RunStrategy::InProcess,
284 let result = rx.recv().unwrap().result;
287 TrFailedMsg("test did not panic as expected".to_string()),
288 "should_panic == {:?}",
294 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
296 let desc = TestDescAndFn {
298 name: StaticTestName("whatever"),
300 #[cfg(not(bootstrap))]
301 ignore_message: None,
302 should_panic: ShouldPanic::No,
305 test_type: TestType::Unknown,
307 testfn: DynTestFn(Box::new(f)),
309 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
311 let test_opts = TestOpts { time_options, ..TestOpts::new() };
312 let (tx, rx) = channel();
313 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
314 let exec_time = rx.recv().unwrap().exec_time;
319 fn test_should_not_report_time() {
320 let exec_time = report_time_test_template(false);
321 assert!(exec_time.is_none());
325 fn test_should_report_time() {
326 let exec_time = report_time_test_template(true);
327 assert!(exec_time.is_some());
330 fn time_test_failure_template(test_type: TestType) -> TestResult {
332 let desc = TestDescAndFn {
334 name: StaticTestName("whatever"),
336 #[cfg(not(bootstrap))]
337 ignore_message: None,
338 should_panic: ShouldPanic::No,
343 testfn: DynTestFn(Box::new(f)),
345 // `Default` will initialize all the thresholds to 0 milliseconds.
346 let mut time_options = TestTimeOptions::default();
347 time_options.error_on_excess = true;
349 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
350 let (tx, rx) = channel();
351 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
352 let result = rx.recv().unwrap().result;
358 fn test_error_on_exceed() {
359 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
361 for test_type in types.iter() {
362 let result = time_test_failure_template(*test_type);
364 assert_eq!(result, TestResult::TrTimedFail);
367 // Check that for unknown tests thresholds aren't applied.
368 let result = time_test_failure_template(TestType::Unknown);
369 assert_eq!(result, TestResult::TrOk);
372 fn typed_test_desc(test_type: TestType) -> TestDesc {
374 name: StaticTestName("whatever"),
376 #[cfg(not(bootstrap))]
377 ignore_message: None,
378 should_panic: ShouldPanic::No,
385 fn test_exec_time(millis: u64) -> TestExecTime {
386 TestExecTime(Duration::from_millis(millis))
390 fn test_time_options_threshold() {
391 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
392 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
393 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
395 let options = TestTimeOptions {
396 error_on_excess: false,
397 unit_threshold: unit.clone(),
398 integration_threshold: integration.clone(),
399 doctest_threshold: doc.clone(),
403 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
404 (TestType::UnitTest, unit.warn.as_millis(), true, false),
405 (TestType::UnitTest, unit.critical.as_millis(), true, true),
406 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
407 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
408 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
409 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
410 (TestType::DocTest, doc.warn.as_millis(), true, false),
411 (TestType::DocTest, doc.critical.as_millis(), true, true),
414 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
415 let test_desc = typed_test_desc(*test_type);
416 let exec_time = test_exec_time(*time as u64);
418 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
419 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
424 fn parse_ignored_flag() {
425 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
426 let opts = parse_opts(&args).unwrap().unwrap();
427 assert_eq!(opts.run_ignored, RunIgnored::Only);
431 fn parse_show_output_flag() {
432 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
433 let opts = parse_opts(&args).unwrap().unwrap();
434 assert!(opts.options.display_output);
438 fn parse_include_ignored_flag() {
439 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
440 let opts = parse_opts(&args).unwrap().unwrap();
441 assert_eq!(opts.run_ignored, RunIgnored::Yes);
445 pub fn filter_for_ignored_option() {
446 // When we run ignored tests the test filter should filter out all the
447 // unignored tests and flip the ignore flag on the rest to false
449 let mut opts = TestOpts::new();
450 opts.run_tests = true;
451 opts.run_ignored = RunIgnored::Only;
453 let tests = one_ignored_one_unignored_test();
454 let filtered = filter_tests(&opts, tests);
456 assert_eq!(filtered.len(), 1);
457 assert_eq!(filtered[0].desc.name.to_string(), "1");
458 assert!(!filtered[0].desc.ignore);
462 pub fn run_include_ignored_option() {
463 // When we "--include-ignored" tests, the ignore flag should be set to false on
464 // all tests and no test filtered out
466 let mut opts = TestOpts::new();
467 opts.run_tests = true;
468 opts.run_ignored = RunIgnored::Yes;
470 let tests = one_ignored_one_unignored_test();
471 let filtered = filter_tests(&opts, tests);
473 assert_eq!(filtered.len(), 2);
474 assert!(!filtered[0].desc.ignore);
475 assert!(!filtered[1].desc.ignore);
479 pub fn exclude_should_panic_option() {
480 let mut opts = TestOpts::new();
481 opts.run_tests = true;
482 opts.exclude_should_panic = true;
484 let mut tests = one_ignored_one_unignored_test();
485 tests.push(TestDescAndFn {
487 name: StaticTestName("3"),
489 #[cfg(not(bootstrap))]
490 ignore_message: None,
491 should_panic: ShouldPanic::Yes,
494 test_type: TestType::Unknown,
496 testfn: DynTestFn(Box::new(move || {})),
499 let filtered = filter_tests(&opts, tests);
501 assert_eq!(filtered.len(), 2);
502 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
506 pub fn exact_filter_match() {
507 fn tests() -> Vec<TestDescAndFn> {
508 ["base", "base::test", "base::test1", "base::test2"]
510 .map(|name| TestDescAndFn {
512 name: StaticTestName(name),
514 #[cfg(not(bootstrap))]
515 ignore_message: None,
516 should_panic: ShouldPanic::No,
519 test_type: TestType::Unknown,
521 testfn: DynTestFn(Box::new(move || {})),
527 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
528 assert_eq!(substr.len(), 4);
531 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
532 assert_eq!(substr.len(), 4);
535 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
536 assert_eq!(substr.len(), 3);
539 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
540 assert_eq!(substr.len(), 3);
542 let substr = filter_tests(
543 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
546 assert_eq!(substr.len(), 2);
548 let exact = filter_tests(
549 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
552 assert_eq!(exact.len(), 1);
554 let exact = filter_tests(
555 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
558 assert_eq!(exact.len(), 0);
560 let exact = filter_tests(
561 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
564 assert_eq!(exact.len(), 0);
566 let exact = filter_tests(
567 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
570 assert_eq!(exact.len(), 1);
572 let exact = filter_tests(
574 filters: vec!["base".into(), "base::test".into()],
580 assert_eq!(exact.len(), 2);
583 fn sample_tests() -> Vec<TestDescAndFn> {
585 "sha1::test".to_string(),
586 "isize::test_to_str".to_string(),
587 "isize::test_pow".to_string(),
588 "test::do_not_run_ignored_tests".to_string(),
589 "test::ignored_tests_result_in_ignored".to_string(),
590 "test::first_free_arg_should_be_a_filter".to_string(),
591 "test::parse_ignored_flag".to_string(),
592 "test::parse_include_ignored_flag".to_string(),
593 "test::filter_for_ignored_option".to_string(),
594 "test::run_include_ignored_option".to_string(),
595 "test::sort_tests".to_string(),
598 let mut tests = Vec::new();
600 let test = TestDescAndFn {
602 name: DynTestName((*name).clone()),
604 #[cfg(not(bootstrap))]
605 ignore_message: None,
606 should_panic: ShouldPanic::No,
609 test_type: TestType::Unknown,
611 testfn: DynTestFn(Box::new(testfn)),
619 pub fn sort_tests() {
620 let mut opts = TestOpts::new();
621 opts.run_tests = true;
623 let tests = sample_tests();
624 let filtered = filter_tests(&opts, tests);
627 "isize::test_pow".to_string(),
628 "isize::test_to_str".to_string(),
629 "sha1::test".to_string(),
630 "test::do_not_run_ignored_tests".to_string(),
631 "test::filter_for_ignored_option".to_string(),
632 "test::first_free_arg_should_be_a_filter".to_string(),
633 "test::ignored_tests_result_in_ignored".to_string(),
634 "test::parse_ignored_flag".to_string(),
635 "test::parse_include_ignored_flag".to_string(),
636 "test::run_include_ignored_option".to_string(),
637 "test::sort_tests".to_string(),
640 for (a, b) in expected.iter().zip(filtered) {
641 assert_eq!(*a, b.desc.name.to_string());
646 pub fn shuffle_tests() {
647 let mut opts = TestOpts::new();
650 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
653 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
655 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
657 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
659 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
661 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
665 pub fn shuffle_tests_with_seed() {
666 let mut opts = TestOpts::new();
669 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
672 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
674 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
676 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
677 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
679 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
683 pub fn order_depends_on_more_than_seed() {
684 let mut opts = TestOpts::new();
687 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
689 let mut left_tests = sample_tests();
690 let mut right_tests = sample_tests();
693 right_tests.remove(0);
696 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
698 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
700 assert_eq!(left.len(), right.len());
702 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
704 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
705 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
707 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
711 pub fn test_metricmap_compare() {
712 let mut m1 = MetricMap::new();
713 let mut m2 = MetricMap::new();
714 m1.insert_metric("in-both-noise", 1000.0, 200.0);
715 m2.insert_metric("in-both-noise", 1100.0, 200.0);
717 m1.insert_metric("in-first-noise", 1000.0, 2.0);
718 m2.insert_metric("in-second-noise", 1000.0, 2.0);
720 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
721 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
723 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
724 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
726 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
727 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
729 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
730 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
734 pub fn test_bench_once_no_iter() {
735 fn f(_: &mut Bencher) {}
740 pub fn test_bench_once_iter() {
741 fn f(b: &mut Bencher) {
748 pub fn test_bench_no_iter() {
749 fn f(_: &mut Bencher) {}
751 let (tx, rx) = channel();
753 let desc = TestDesc {
754 name: StaticTestName("f"),
756 #[cfg(not(bootstrap))]
757 ignore_message: None,
758 should_panic: ShouldPanic::No,
761 test_type: TestType::Unknown,
764 crate::bench::benchmark(TestId(0), desc, tx, true, f);
769 pub fn test_bench_iter() {
770 fn f(b: &mut Bencher) {
774 let (tx, rx) = channel();
776 let desc = TestDesc {
777 name: StaticTestName("f"),
779 #[cfg(not(bootstrap))]
780 ignore_message: None,
781 should_panic: ShouldPanic::No,
784 test_type: TestType::Unknown,
787 crate::bench::benchmark(TestId(0), desc, tx, true, f);
792 fn should_sort_failures_before_printing_them() {
793 let test_a = TestDesc {
794 name: StaticTestName("a"),
796 #[cfg(not(bootstrap))]
797 ignore_message: None,
798 should_panic: ShouldPanic::No,
801 test_type: TestType::Unknown,
804 let test_b = TestDesc {
805 name: StaticTestName("b"),
807 #[cfg(not(bootstrap))]
808 ignore_message: None,
809 should_panic: ShouldPanic::No,
812 test_type: TestType::Unknown,
815 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
817 let st = console::ConsoleTestState {
826 metrics: MetricMap::new(),
827 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
828 options: Options::new(),
829 not_failures: Vec::new(),
830 time_failures: Vec::new(),
833 out.write_failures(&st).unwrap();
834 let s = match out.output_location() {
835 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
836 &OutputLocation::Pretty(_) => unreachable!(),
839 let apos = s.find("a").unwrap();
840 let bpos = s.find("b").unwrap();
841 assert!(apos < bpos);