5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
53 options: Options::new(),
59 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
63 name: StaticTestName("1"),
66 should_panic: ShouldPanic::No,
69 test_type: TestType::Unknown,
71 testfn: DynTestFn(Box::new(move || Ok(()))),
75 name: StaticTestName("2"),
78 should_panic: ShouldPanic::No,
81 test_type: TestType::Unknown,
83 testfn: DynTestFn(Box::new(move || Ok(()))),
89 pub fn do_not_run_ignored_tests() {
90 fn f() -> Result<(), String> {
93 let desc = TestDescAndFn {
95 name: StaticTestName("whatever"),
98 should_panic: ShouldPanic::No,
101 test_type: TestType::Unknown,
103 testfn: DynTestFn(Box::new(f)),
105 let (tx, rx) = channel();
106 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
107 let result = rx.recv().unwrap().result;
108 assert_ne!(result, TrOk);
112 pub fn ignored_tests_result_in_ignored() {
113 fn f() -> Result<(), String> {
116 let desc = TestDescAndFn {
118 name: StaticTestName("whatever"),
120 ignore_message: None,
121 should_panic: ShouldPanic::No,
124 test_type: TestType::Unknown,
126 testfn: DynTestFn(Box::new(f)),
128 let (tx, rx) = channel();
129 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
130 let result = rx.recv().unwrap().result;
131 assert_eq!(result, TrIgnored);
134 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
136 #[cfg(not(target_os = "emscripten"))]
137 fn test_should_panic() {
138 fn f() -> Result<(), String> {
141 let desc = TestDescAndFn {
143 name: StaticTestName("whatever"),
145 ignore_message: None,
146 should_panic: ShouldPanic::Yes,
149 test_type: TestType::Unknown,
151 testfn: DynTestFn(Box::new(f)),
153 let (tx, rx) = channel();
154 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
155 let result = rx.recv().unwrap().result;
156 assert_eq!(result, TrOk);
159 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
161 #[cfg(not(target_os = "emscripten"))]
162 fn test_should_panic_good_message() {
163 fn f() -> Result<(), String> {
164 panic!("an error message");
166 let desc = TestDescAndFn {
168 name: StaticTestName("whatever"),
170 ignore_message: None,
171 should_panic: ShouldPanic::YesWithMessage("error message"),
174 test_type: TestType::Unknown,
176 testfn: DynTestFn(Box::new(f)),
178 let (tx, rx) = channel();
179 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
180 let result = rx.recv().unwrap().result;
181 assert_eq!(result, TrOk);
184 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
186 #[cfg(not(target_os = "emscripten"))]
187 fn test_should_panic_bad_message() {
188 use crate::tests::TrFailedMsg;
189 fn f() -> Result<(), String> {
190 panic!("an error message");
192 let expected = "foobar";
193 let failed_msg = r#"panic did not contain expected string
194 panic message: `"an error message"`,
195 expected substring: `"foobar"`"#;
196 let desc = TestDescAndFn {
198 name: StaticTestName("whatever"),
200 ignore_message: None,
201 should_panic: ShouldPanic::YesWithMessage(expected),
204 test_type: TestType::Unknown,
206 testfn: DynTestFn(Box::new(f)),
208 let (tx, rx) = channel();
209 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
210 let result = rx.recv().unwrap().result;
211 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
214 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
216 #[cfg(not(target_os = "emscripten"))]
217 fn test_should_panic_non_string_message_type() {
218 use crate::tests::TrFailedMsg;
219 use std::any::TypeId;
220 fn f() -> Result<(), String> {
221 std::panic::panic_any(1i32);
223 let expected = "foobar";
224 let failed_msg = format!(
225 r#"expected panic with string value,
226 found non-string value: `{:?}`
227 expected substring: `"foobar"`"#,
230 let desc = TestDescAndFn {
232 name: StaticTestName("whatever"),
234 ignore_message: None,
235 should_panic: ShouldPanic::YesWithMessage(expected),
238 test_type: TestType::Unknown,
240 testfn: DynTestFn(Box::new(f)),
242 let (tx, rx) = channel();
243 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
244 let result = rx.recv().unwrap().result;
245 assert_eq!(result, TrFailedMsg(failed_msg));
248 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
250 #[cfg(not(target_os = "emscripten"))]
251 fn test_should_panic_but_succeeds() {
252 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
254 for &should_panic in should_panic_variants.iter() {
255 fn f() -> Result<(), String> {
258 let desc = TestDescAndFn {
260 name: StaticTestName("whatever"),
262 ignore_message: None,
266 test_type: TestType::Unknown,
268 testfn: DynTestFn(Box::new(f)),
270 let (tx, rx) = channel();
271 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx);
272 let result = rx.recv().unwrap().result;
275 TrFailedMsg("test did not panic as expected".to_string()),
276 "should_panic == {:?}",
282 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
283 fn f() -> Result<(), String> {
286 let desc = TestDescAndFn {
288 name: StaticTestName("whatever"),
290 ignore_message: None,
291 should_panic: ShouldPanic::No,
294 test_type: TestType::Unknown,
296 testfn: DynTestFn(Box::new(f)),
298 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
300 let test_opts = TestOpts { time_options, ..TestOpts::new() };
301 let (tx, rx) = channel();
302 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx);
303 let exec_time = rx.recv().unwrap().exec_time;
308 fn test_should_not_report_time() {
309 let exec_time = report_time_test_template(false);
310 assert!(exec_time.is_none());
314 fn test_should_report_time() {
315 let exec_time = report_time_test_template(true);
316 assert!(exec_time.is_some());
319 fn time_test_failure_template(test_type: TestType) -> TestResult {
320 fn f() -> Result<(), String> {
323 let desc = TestDescAndFn {
325 name: StaticTestName("whatever"),
327 ignore_message: None,
328 should_panic: ShouldPanic::No,
333 testfn: DynTestFn(Box::new(f)),
335 // `Default` will initialize all the thresholds to 0 milliseconds.
336 let mut time_options = TestTimeOptions::default();
337 time_options.error_on_excess = true;
339 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
340 let (tx, rx) = channel();
341 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx);
342 let result = rx.recv().unwrap().result;
348 fn test_error_on_exceed() {
349 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
351 for test_type in types.iter() {
352 let result = time_test_failure_template(*test_type);
354 assert_eq!(result, TestResult::TrTimedFail);
357 // Check that for unknown tests thresholds aren't applied.
358 let result = time_test_failure_template(TestType::Unknown);
359 assert_eq!(result, TestResult::TrOk);
362 fn typed_test_desc(test_type: TestType) -> TestDesc {
364 name: StaticTestName("whatever"),
366 ignore_message: None,
367 should_panic: ShouldPanic::No,
374 fn test_exec_time(millis: u64) -> TestExecTime {
375 TestExecTime(Duration::from_millis(millis))
379 fn test_time_options_threshold() {
380 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
381 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
382 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
384 let options = TestTimeOptions {
385 error_on_excess: false,
386 unit_threshold: unit.clone(),
387 integration_threshold: integration.clone(),
388 doctest_threshold: doc.clone(),
392 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
393 (TestType::UnitTest, unit.warn.as_millis(), true, false),
394 (TestType::UnitTest, unit.critical.as_millis(), true, true),
395 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
396 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
397 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
398 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
399 (TestType::DocTest, doc.warn.as_millis(), true, false),
400 (TestType::DocTest, doc.critical.as_millis(), true, true),
403 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
404 let test_desc = typed_test_desc(*test_type);
405 let exec_time = test_exec_time(*time as u64);
407 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
408 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
413 fn parse_ignored_flag() {
414 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
415 let opts = parse_opts(&args).unwrap().unwrap();
416 assert_eq!(opts.run_ignored, RunIgnored::Only);
420 fn parse_show_output_flag() {
421 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
422 let opts = parse_opts(&args).unwrap().unwrap();
423 assert!(opts.options.display_output);
427 fn parse_include_ignored_flag() {
428 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
429 let opts = parse_opts(&args).unwrap().unwrap();
430 assert_eq!(opts.run_ignored, RunIgnored::Yes);
434 pub fn filter_for_ignored_option() {
435 // When we run ignored tests the test filter should filter out all the
436 // unignored tests and flip the ignore flag on the rest to false
438 let mut opts = TestOpts::new();
439 opts.run_tests = true;
440 opts.run_ignored = RunIgnored::Only;
442 let tests = one_ignored_one_unignored_test();
443 let filtered = filter_tests(&opts, tests);
445 assert_eq!(filtered.len(), 1);
446 assert_eq!(filtered[0].desc.name.to_string(), "1");
447 assert!(!filtered[0].desc.ignore);
451 pub fn run_include_ignored_option() {
452 // When we "--include-ignored" tests, the ignore flag should be set to false on
453 // all tests and no test filtered out
455 let mut opts = TestOpts::new();
456 opts.run_tests = true;
457 opts.run_ignored = RunIgnored::Yes;
459 let tests = one_ignored_one_unignored_test();
460 let filtered = filter_tests(&opts, tests);
462 assert_eq!(filtered.len(), 2);
463 assert!(!filtered[0].desc.ignore);
464 assert!(!filtered[1].desc.ignore);
468 pub fn exclude_should_panic_option() {
469 let mut opts = TestOpts::new();
470 opts.run_tests = true;
471 opts.exclude_should_panic = true;
473 let mut tests = one_ignored_one_unignored_test();
474 tests.push(TestDescAndFn {
476 name: StaticTestName("3"),
478 ignore_message: None,
479 should_panic: ShouldPanic::Yes,
482 test_type: TestType::Unknown,
484 testfn: DynTestFn(Box::new(move || Ok(()))),
487 let filtered = filter_tests(&opts, tests);
489 assert_eq!(filtered.len(), 2);
490 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
494 pub fn exact_filter_match() {
495 fn tests() -> Vec<TestDescAndFn> {
496 ["base", "base::test", "base::test1", "base::test2"]
498 .map(|name| TestDescAndFn {
500 name: StaticTestName(name),
502 ignore_message: None,
503 should_panic: ShouldPanic::No,
506 test_type: TestType::Unknown,
508 testfn: DynTestFn(Box::new(move || Ok(()))),
514 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
515 assert_eq!(substr.len(), 4);
518 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
519 assert_eq!(substr.len(), 4);
522 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
523 assert_eq!(substr.len(), 3);
526 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
527 assert_eq!(substr.len(), 3);
529 let substr = filter_tests(
530 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
533 assert_eq!(substr.len(), 2);
535 let exact = filter_tests(
536 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
539 assert_eq!(exact.len(), 1);
541 let exact = filter_tests(
542 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
545 assert_eq!(exact.len(), 0);
547 let exact = filter_tests(
548 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
551 assert_eq!(exact.len(), 0);
553 let exact = filter_tests(
554 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
557 assert_eq!(exact.len(), 1);
559 let exact = filter_tests(
561 filters: vec!["base".into(), "base::test".into()],
567 assert_eq!(exact.len(), 2);
570 fn sample_tests() -> Vec<TestDescAndFn> {
572 "sha1::test".to_string(),
573 "isize::test_to_str".to_string(),
574 "isize::test_pow".to_string(),
575 "test::do_not_run_ignored_tests".to_string(),
576 "test::ignored_tests_result_in_ignored".to_string(),
577 "test::first_free_arg_should_be_a_filter".to_string(),
578 "test::parse_ignored_flag".to_string(),
579 "test::parse_include_ignored_flag".to_string(),
580 "test::filter_for_ignored_option".to_string(),
581 "test::run_include_ignored_option".to_string(),
582 "test::sort_tests".to_string(),
584 fn testfn() -> Result<(), String> {
587 let mut tests = Vec::new();
589 let test = TestDescAndFn {
591 name: DynTestName((*name).clone()),
593 ignore_message: None,
594 should_panic: ShouldPanic::No,
597 test_type: TestType::Unknown,
599 testfn: DynTestFn(Box::new(testfn)),
607 pub fn shuffle_tests() {
608 let mut opts = TestOpts::new();
611 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
614 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
616 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
618 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
620 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
622 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
626 pub fn shuffle_tests_with_seed() {
627 let mut opts = TestOpts::new();
630 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
633 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
635 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
637 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
638 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
640 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
644 pub fn order_depends_on_more_than_seed() {
645 let mut opts = TestOpts::new();
648 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
650 let mut left_tests = sample_tests();
651 let mut right_tests = sample_tests();
654 right_tests.remove(0);
657 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
659 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
661 assert_eq!(left.len(), right.len());
663 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
665 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
666 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
668 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
672 pub fn test_metricmap_compare() {
673 let mut m1 = MetricMap::new();
674 let mut m2 = MetricMap::new();
675 m1.insert_metric("in-both-noise", 1000.0, 200.0);
676 m2.insert_metric("in-both-noise", 1100.0, 200.0);
678 m1.insert_metric("in-first-noise", 1000.0, 2.0);
679 m2.insert_metric("in-second-noise", 1000.0, 2.0);
681 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
682 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
684 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
685 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
687 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
688 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
690 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
691 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
695 pub fn test_bench_once_no_iter() {
696 fn f(_: &mut Bencher) -> Result<(), String> {
699 bench::run_once(f).unwrap();
703 pub fn test_bench_once_iter() {
704 fn f(b: &mut Bencher) -> Result<(), String> {
708 bench::run_once(f).unwrap();
712 pub fn test_bench_no_iter() {
713 fn f(_: &mut Bencher) -> Result<(), String> {
717 let (tx, rx) = channel();
719 let desc = TestDesc {
720 name: StaticTestName("f"),
722 ignore_message: None,
723 should_panic: ShouldPanic::No,
726 test_type: TestType::Unknown,
729 crate::bench::benchmark(TestId(0), desc, tx, true, f);
734 pub fn test_bench_iter() {
735 fn f(b: &mut Bencher) -> Result<(), String> {
740 let (tx, rx) = channel();
742 let desc = TestDesc {
743 name: StaticTestName("f"),
745 ignore_message: None,
746 should_panic: ShouldPanic::No,
749 test_type: TestType::Unknown,
752 crate::bench::benchmark(TestId(0), desc, tx, true, f);
757 fn should_sort_failures_before_printing_them() {
758 let test_a = TestDesc {
759 name: StaticTestName("a"),
761 ignore_message: None,
762 should_panic: ShouldPanic::No,
765 test_type: TestType::Unknown,
768 let test_b = TestDesc {
769 name: StaticTestName("b"),
771 ignore_message: None,
772 should_panic: ShouldPanic::No,
775 test_type: TestType::Unknown,
778 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
780 let st = console::ConsoleTestState {
789 metrics: MetricMap::new(),
790 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
791 options: Options::new(),
792 not_failures: Vec::new(),
793 time_failures: Vec::new(),
796 out.write_failures(&st).unwrap();
797 let s = match out.output_location() {
798 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
799 &OutputLocation::Pretty(_) => unreachable!(),
802 let apos = s.find("a").unwrap();
803 let bpos = s.find("b").unwrap();
804 assert!(apos < bpos);
808 #[cfg(not(target_os = "emscripten"))]
809 fn test_dyn_bench_returning_err_fails_when_run_as_test() {
810 fn f(_: &mut Bencher) -> Result<(), String> {
811 Result::Err("An error".into())
813 let desc = TestDescAndFn {
815 name: StaticTestName("whatever"),
817 ignore_message: None,
818 should_panic: ShouldPanic::No,
821 test_type: TestType::Unknown,
823 testfn: DynBenchFn(Box::new(f)),
825 let (tx, rx) = channel();
826 let notify = move |event: TestEvent| {
827 if let TestEvent::TeResult(result) = event {
828 tx.send(result).unwrap();
832 run_tests(&TestOpts { run_tests: true, ..TestOpts::new() }, vec![desc], notify).unwrap();
833 let result = rx.recv().unwrap().result;
834 assert_eq!(result, TrFailed);