5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
53 options: Options::new(),
58 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
62 name: StaticTestName("1"),
65 should_panic: ShouldPanic::No,
68 test_type: TestType::Unknown,
70 testfn: DynTestFn(Box::new(move || {})),
74 name: StaticTestName("2"),
77 should_panic: ShouldPanic::No,
80 test_type: TestType::Unknown,
82 testfn: DynTestFn(Box::new(move || {})),
88 pub fn do_not_run_ignored_tests() {
92 let desc = TestDescAndFn {
94 name: StaticTestName("whatever"),
97 should_panic: ShouldPanic::No,
100 test_type: TestType::Unknown,
102 testfn: DynTestFn(Box::new(f)),
104 let (tx, rx) = channel();
105 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
106 let result = rx.recv().unwrap().result;
107 assert_ne!(result, TrOk);
111 pub fn ignored_tests_result_in_ignored() {
113 let desc = TestDescAndFn {
115 name: StaticTestName("whatever"),
117 ignore_message: None,
118 should_panic: ShouldPanic::No,
121 test_type: TestType::Unknown,
123 testfn: DynTestFn(Box::new(f)),
125 let (tx, rx) = channel();
126 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
127 let result = rx.recv().unwrap().result;
128 assert_eq!(result, TrIgnored);
131 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
133 #[cfg(not(target_os = "emscripten"))]
134 fn test_should_panic() {
138 let desc = TestDescAndFn {
140 name: StaticTestName("whatever"),
142 ignore_message: None,
143 should_panic: ShouldPanic::Yes,
146 test_type: TestType::Unknown,
148 testfn: DynTestFn(Box::new(f)),
150 let (tx, rx) = channel();
151 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
152 let result = rx.recv().unwrap().result;
153 assert_eq!(result, TrOk);
156 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
158 #[cfg(not(target_os = "emscripten"))]
159 fn test_should_panic_good_message() {
161 panic!("an error message");
163 let desc = TestDescAndFn {
165 name: StaticTestName("whatever"),
167 ignore_message: None,
168 should_panic: ShouldPanic::YesWithMessage("error message"),
171 test_type: TestType::Unknown,
173 testfn: DynTestFn(Box::new(f)),
175 let (tx, rx) = channel();
176 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
177 let result = rx.recv().unwrap().result;
178 assert_eq!(result, TrOk);
181 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
183 #[cfg(not(target_os = "emscripten"))]
184 fn test_should_panic_bad_message() {
185 use crate::tests::TrFailedMsg;
187 panic!("an error message");
189 let expected = "foobar";
190 let failed_msg = r#"panic did not contain expected string
191 panic message: `"an error message"`,
192 expected substring: `"foobar"`"#;
193 let desc = TestDescAndFn {
195 name: StaticTestName("whatever"),
197 ignore_message: None,
198 should_panic: ShouldPanic::YesWithMessage(expected),
201 test_type: TestType::Unknown,
203 testfn: DynTestFn(Box::new(f)),
205 let (tx, rx) = channel();
206 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
207 let result = rx.recv().unwrap().result;
208 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
211 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
213 #[cfg(not(target_os = "emscripten"))]
214 fn test_should_panic_non_string_message_type() {
215 use crate::tests::TrFailedMsg;
216 use std::any::TypeId;
218 std::panic::panic_any(1i32);
220 let expected = "foobar";
221 let failed_msg = format!(
222 r#"expected panic with string value,
223 found non-string value: `{:?}`
224 expected substring: `"foobar"`"#,
227 let desc = TestDescAndFn {
229 name: StaticTestName("whatever"),
231 ignore_message: None,
232 should_panic: ShouldPanic::YesWithMessage(expected),
235 test_type: TestType::Unknown,
237 testfn: DynTestFn(Box::new(f)),
239 let (tx, rx) = channel();
240 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
241 let result = rx.recv().unwrap().result;
242 assert_eq!(result, TrFailedMsg(failed_msg));
245 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
247 #[cfg(not(target_os = "emscripten"))]
248 fn test_should_panic_but_succeeds() {
249 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
251 for &should_panic in should_panic_variants.iter() {
253 let desc = TestDescAndFn {
255 name: StaticTestName("whatever"),
257 ignore_message: None,
261 test_type: TestType::Unknown,
263 testfn: DynTestFn(Box::new(f)),
265 let (tx, rx) = channel();
271 RunStrategy::InProcess,
275 let result = rx.recv().unwrap().result;
278 TrFailedMsg("test did not panic as expected".to_string()),
279 "should_panic == {:?}",
285 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
287 let desc = TestDescAndFn {
289 name: StaticTestName("whatever"),
291 ignore_message: None,
292 should_panic: ShouldPanic::No,
295 test_type: TestType::Unknown,
297 testfn: DynTestFn(Box::new(f)),
299 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
301 let test_opts = TestOpts { time_options, ..TestOpts::new() };
302 let (tx, rx) = channel();
303 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
304 let exec_time = rx.recv().unwrap().exec_time;
309 fn test_should_not_report_time() {
310 let exec_time = report_time_test_template(false);
311 assert!(exec_time.is_none());
315 fn test_should_report_time() {
316 let exec_time = report_time_test_template(true);
317 assert!(exec_time.is_some());
320 fn time_test_failure_template(test_type: TestType) -> TestResult {
322 let desc = TestDescAndFn {
324 name: StaticTestName("whatever"),
326 ignore_message: None,
327 should_panic: ShouldPanic::No,
332 testfn: DynTestFn(Box::new(f)),
334 // `Default` will initialize all the thresholds to 0 milliseconds.
335 let mut time_options = TestTimeOptions::default();
336 time_options.error_on_excess = true;
338 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
339 let (tx, rx) = channel();
340 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
341 let result = rx.recv().unwrap().result;
347 fn test_error_on_exceed() {
348 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
350 for test_type in types.iter() {
351 let result = time_test_failure_template(*test_type);
353 assert_eq!(result, TestResult::TrTimedFail);
356 // Check that for unknown tests thresholds aren't applied.
357 let result = time_test_failure_template(TestType::Unknown);
358 assert_eq!(result, TestResult::TrOk);
361 fn typed_test_desc(test_type: TestType) -> TestDesc {
363 name: StaticTestName("whatever"),
365 ignore_message: None,
366 should_panic: ShouldPanic::No,
373 fn test_exec_time(millis: u64) -> TestExecTime {
374 TestExecTime(Duration::from_millis(millis))
378 fn test_time_options_threshold() {
379 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
380 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
381 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
383 let options = TestTimeOptions {
384 error_on_excess: false,
385 unit_threshold: unit.clone(),
386 integration_threshold: integration.clone(),
387 doctest_threshold: doc.clone(),
391 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
392 (TestType::UnitTest, unit.warn.as_millis(), true, false),
393 (TestType::UnitTest, unit.critical.as_millis(), true, true),
394 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
395 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
396 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
397 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
398 (TestType::DocTest, doc.warn.as_millis(), true, false),
399 (TestType::DocTest, doc.critical.as_millis(), true, true),
402 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
403 let test_desc = typed_test_desc(*test_type);
404 let exec_time = test_exec_time(*time as u64);
406 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
407 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
412 fn parse_ignored_flag() {
413 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
414 let opts = parse_opts(&args).unwrap().unwrap();
415 assert_eq!(opts.run_ignored, RunIgnored::Only);
419 fn parse_show_output_flag() {
420 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
421 let opts = parse_opts(&args).unwrap().unwrap();
422 assert!(opts.options.display_output);
426 fn parse_include_ignored_flag() {
427 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
428 let opts = parse_opts(&args).unwrap().unwrap();
429 assert_eq!(opts.run_ignored, RunIgnored::Yes);
433 pub fn filter_for_ignored_option() {
434 // When we run ignored tests the test filter should filter out all the
435 // unignored tests and flip the ignore flag on the rest to false
437 let mut opts = TestOpts::new();
438 opts.run_tests = true;
439 opts.run_ignored = RunIgnored::Only;
441 let tests = one_ignored_one_unignored_test();
442 let filtered = filter_tests(&opts, tests);
444 assert_eq!(filtered.len(), 1);
445 assert_eq!(filtered[0].desc.name.to_string(), "1");
446 assert!(!filtered[0].desc.ignore);
450 pub fn run_include_ignored_option() {
451 // When we "--include-ignored" tests, the ignore flag should be set to false on
452 // all tests and no test filtered out
454 let mut opts = TestOpts::new();
455 opts.run_tests = true;
456 opts.run_ignored = RunIgnored::Yes;
458 let tests = one_ignored_one_unignored_test();
459 let filtered = filter_tests(&opts, tests);
461 assert_eq!(filtered.len(), 2);
462 assert!(!filtered[0].desc.ignore);
463 assert!(!filtered[1].desc.ignore);
467 pub fn exclude_should_panic_option() {
468 let mut opts = TestOpts::new();
469 opts.run_tests = true;
470 opts.exclude_should_panic = true;
472 let mut tests = one_ignored_one_unignored_test();
473 tests.push(TestDescAndFn {
475 name: StaticTestName("3"),
477 ignore_message: None,
478 should_panic: ShouldPanic::Yes,
481 test_type: TestType::Unknown,
483 testfn: DynTestFn(Box::new(move || {})),
486 let filtered = filter_tests(&opts, tests);
488 assert_eq!(filtered.len(), 2);
489 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
493 pub fn exact_filter_match() {
494 fn tests() -> Vec<TestDescAndFn> {
495 ["base", "base::test", "base::test1", "base::test2"]
497 .map(|name| TestDescAndFn {
499 name: StaticTestName(name),
501 ignore_message: None,
502 should_panic: ShouldPanic::No,
505 test_type: TestType::Unknown,
507 testfn: DynTestFn(Box::new(move || {})),
513 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
514 assert_eq!(substr.len(), 4);
517 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
518 assert_eq!(substr.len(), 4);
521 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
522 assert_eq!(substr.len(), 3);
525 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
526 assert_eq!(substr.len(), 3);
528 let substr = filter_tests(
529 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
532 assert_eq!(substr.len(), 2);
534 let exact = filter_tests(
535 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
538 assert_eq!(exact.len(), 1);
540 let exact = filter_tests(
541 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
544 assert_eq!(exact.len(), 0);
546 let exact = filter_tests(
547 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
550 assert_eq!(exact.len(), 0);
552 let exact = filter_tests(
553 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
556 assert_eq!(exact.len(), 1);
558 let exact = filter_tests(
560 filters: vec!["base".into(), "base::test".into()],
566 assert_eq!(exact.len(), 2);
569 fn sample_tests() -> Vec<TestDescAndFn> {
571 "sha1::test".to_string(),
572 "isize::test_to_str".to_string(),
573 "isize::test_pow".to_string(),
574 "test::do_not_run_ignored_tests".to_string(),
575 "test::ignored_tests_result_in_ignored".to_string(),
576 "test::first_free_arg_should_be_a_filter".to_string(),
577 "test::parse_ignored_flag".to_string(),
578 "test::parse_include_ignored_flag".to_string(),
579 "test::filter_for_ignored_option".to_string(),
580 "test::run_include_ignored_option".to_string(),
581 "test::sort_tests".to_string(),
584 let mut tests = Vec::new();
586 let test = TestDescAndFn {
588 name: DynTestName((*name).clone()),
590 ignore_message: None,
591 should_panic: ShouldPanic::No,
594 test_type: TestType::Unknown,
596 testfn: DynTestFn(Box::new(testfn)),
604 pub fn sort_tests() {
605 let mut opts = TestOpts::new();
606 opts.run_tests = true;
608 let tests = sample_tests();
609 let filtered = filter_tests(&opts, tests);
612 "isize::test_pow".to_string(),
613 "isize::test_to_str".to_string(),
614 "sha1::test".to_string(),
615 "test::do_not_run_ignored_tests".to_string(),
616 "test::filter_for_ignored_option".to_string(),
617 "test::first_free_arg_should_be_a_filter".to_string(),
618 "test::ignored_tests_result_in_ignored".to_string(),
619 "test::parse_ignored_flag".to_string(),
620 "test::parse_include_ignored_flag".to_string(),
621 "test::run_include_ignored_option".to_string(),
622 "test::sort_tests".to_string(),
625 for (a, b) in expected.iter().zip(filtered) {
626 assert_eq!(*a, b.desc.name.to_string());
631 pub fn shuffle_tests() {
632 let mut opts = TestOpts::new();
635 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
638 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
640 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
642 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
644 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
646 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
650 pub fn shuffle_tests_with_seed() {
651 let mut opts = TestOpts::new();
654 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
657 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
659 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
661 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
662 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
664 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
668 pub fn order_depends_on_more_than_seed() {
669 let mut opts = TestOpts::new();
672 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
674 let mut left_tests = sample_tests();
675 let mut right_tests = sample_tests();
678 right_tests.remove(0);
681 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
683 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
685 assert_eq!(left.len(), right.len());
687 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
689 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
690 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
692 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
696 pub fn test_metricmap_compare() {
697 let mut m1 = MetricMap::new();
698 let mut m2 = MetricMap::new();
699 m1.insert_metric("in-both-noise", 1000.0, 200.0);
700 m2.insert_metric("in-both-noise", 1100.0, 200.0);
702 m1.insert_metric("in-first-noise", 1000.0, 2.0);
703 m2.insert_metric("in-second-noise", 1000.0, 2.0);
705 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
706 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
708 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
709 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
711 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
712 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
714 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
715 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
719 pub fn test_bench_once_no_iter() {
720 fn f(_: &mut Bencher) {}
725 pub fn test_bench_once_iter() {
726 fn f(b: &mut Bencher) {
733 pub fn test_bench_no_iter() {
734 fn f(_: &mut Bencher) {}
736 let (tx, rx) = channel();
738 let desc = TestDesc {
739 name: StaticTestName("f"),
741 ignore_message: None,
742 should_panic: ShouldPanic::No,
745 test_type: TestType::Unknown,
748 crate::bench::benchmark(TestId(0), desc, tx, true, f);
753 pub fn test_bench_iter() {
754 fn f(b: &mut Bencher) {
758 let (tx, rx) = channel();
760 let desc = TestDesc {
761 name: StaticTestName("f"),
763 ignore_message: None,
764 should_panic: ShouldPanic::No,
767 test_type: TestType::Unknown,
770 crate::bench::benchmark(TestId(0), desc, tx, true, f);
775 fn should_sort_failures_before_printing_them() {
776 let test_a = TestDesc {
777 name: StaticTestName("a"),
779 ignore_message: None,
780 should_panic: ShouldPanic::No,
783 test_type: TestType::Unknown,
786 let test_b = TestDesc {
787 name: StaticTestName("b"),
789 ignore_message: None,
790 should_panic: ShouldPanic::No,
793 test_type: TestType::Unknown,
796 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
798 let st = console::ConsoleTestState {
807 metrics: MetricMap::new(),
808 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
809 options: Options::new(),
810 not_failures: Vec::new(),
811 time_failures: Vec::new(),
814 out.write_failures(&st).unwrap();
815 let s = match out.output_location() {
816 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
817 &OutputLocation::Pretty(_) => unreachable!(),
820 let apos = s.find("a").unwrap();
821 let bpos = s.find("b").unwrap();
822 assert!(apos < bpos);