5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
51 options: Options::new(),
56 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
60 name: StaticTestName("1"),
62 should_panic: ShouldPanic::No,
66 test_type: TestType::Unknown,
68 testfn: DynTestFn(Box::new(move || {})),
72 name: StaticTestName("2"),
74 should_panic: ShouldPanic::No,
78 test_type: TestType::Unknown,
80 testfn: DynTestFn(Box::new(move || {})),
86 pub fn do_not_run_ignored_tests() {
90 let desc = TestDescAndFn {
92 name: StaticTestName("whatever"),
94 should_panic: ShouldPanic::No,
98 test_type: TestType::Unknown,
100 testfn: DynTestFn(Box::new(f)),
102 let (tx, rx) = channel();
103 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
104 let result = rx.recv().unwrap().result;
105 assert_ne!(result, TrOk);
109 pub fn ignored_tests_result_in_ignored() {
111 let desc = TestDescAndFn {
113 name: StaticTestName("whatever"),
115 should_panic: ShouldPanic::No,
119 test_type: TestType::Unknown,
121 testfn: DynTestFn(Box::new(f)),
123 let (tx, rx) = channel();
124 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
125 let result = rx.recv().unwrap().result;
126 assert_eq!(result, TrIgnored);
129 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
131 #[cfg(not(target_os = "emscripten"))]
132 fn test_should_panic() {
136 let desc = TestDescAndFn {
138 name: StaticTestName("whatever"),
140 should_panic: ShouldPanic::Yes,
144 test_type: TestType::Unknown,
146 testfn: DynTestFn(Box::new(f)),
148 let (tx, rx) = channel();
149 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
150 let result = rx.recv().unwrap().result;
151 assert_eq!(result, TrOk);
154 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
156 #[cfg(not(target_os = "emscripten"))]
157 fn test_should_panic_good_message() {
159 panic!("an error message");
161 let desc = TestDescAndFn {
163 name: StaticTestName("whatever"),
165 should_panic: ShouldPanic::YesWithMessage("error message"),
169 test_type: TestType::Unknown,
171 testfn: DynTestFn(Box::new(f)),
173 let (tx, rx) = channel();
174 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
175 let result = rx.recv().unwrap().result;
176 assert_eq!(result, TrOk);
179 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
181 #[cfg(not(target_os = "emscripten"))]
182 fn test_should_panic_bad_message() {
183 use crate::tests::TrFailedMsg;
185 panic!("an error message");
187 let expected = "foobar";
188 let failed_msg = r#"panic did not contain expected string
189 panic message: `"an error message"`,
190 expected substring: `"foobar"`"#;
191 let desc = TestDescAndFn {
193 name: StaticTestName("whatever"),
195 should_panic: ShouldPanic::YesWithMessage(expected),
199 test_type: TestType::Unknown,
201 testfn: DynTestFn(Box::new(f)),
203 let (tx, rx) = channel();
204 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
205 let result = rx.recv().unwrap().result;
206 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
209 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
211 #[cfg(not(target_os = "emscripten"))]
212 fn test_should_panic_non_string_message_type() {
213 use crate::tests::TrFailedMsg;
214 use std::any::TypeId;
216 std::panic::panic_any(1i32);
218 let expected = "foobar";
219 let failed_msg = format!(
220 r#"expected panic with string value,
221 found non-string value: `{:?}`
222 expected substring: `"foobar"`"#,
225 let desc = TestDescAndFn {
227 name: StaticTestName("whatever"),
229 should_panic: ShouldPanic::YesWithMessage(expected),
233 test_type: TestType::Unknown,
235 testfn: DynTestFn(Box::new(f)),
237 let (tx, rx) = channel();
238 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
239 let result = rx.recv().unwrap().result;
240 assert_eq!(result, TrFailedMsg(failed_msg));
243 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
245 #[cfg(not(target_os = "emscripten"))]
246 fn test_should_panic_but_succeeds() {
247 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
249 for &should_panic in should_panic_variants.iter() {
251 let desc = TestDescAndFn {
253 name: StaticTestName("whatever"),
259 test_type: TestType::Unknown,
261 testfn: DynTestFn(Box::new(f)),
263 let (tx, rx) = channel();
269 RunStrategy::InProcess,
273 let result = rx.recv().unwrap().result;
276 TrFailedMsg("test did not panic as expected".to_string()),
277 "should_panic == {:?}",
283 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
285 let desc = TestDescAndFn {
287 name: StaticTestName("whatever"),
289 should_panic: ShouldPanic::No,
293 test_type: TestType::Unknown,
295 testfn: DynTestFn(Box::new(f)),
297 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
299 let test_opts = TestOpts { time_options, ..TestOpts::new() };
300 let (tx, rx) = channel();
301 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
302 let exec_time = rx.recv().unwrap().exec_time;
307 fn test_should_not_report_time() {
308 let exec_time = report_time_test_template(false);
309 assert!(exec_time.is_none());
313 fn test_should_report_time() {
314 let exec_time = report_time_test_template(true);
315 assert!(exec_time.is_some());
318 fn time_test_failure_template(test_type: TestType) -> TestResult {
320 let desc = TestDescAndFn {
322 name: StaticTestName("whatever"),
324 should_panic: ShouldPanic::No,
330 testfn: DynTestFn(Box::new(f)),
332 // `Default` will initialize all the thresholds to 0 milliseconds.
333 let mut time_options = TestTimeOptions::default();
334 time_options.error_on_excess = true;
336 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
337 let (tx, rx) = channel();
338 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
339 let result = rx.recv().unwrap().result;
345 fn test_error_on_exceed() {
346 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
348 for test_type in types.iter() {
349 let result = time_test_failure_template(*test_type);
351 assert_eq!(result, TestResult::TrTimedFail);
354 // Check that for unknown tests thresholds aren't applied.
355 let result = time_test_failure_template(TestType::Unknown);
356 assert_eq!(result, TestResult::TrOk);
359 fn typed_test_desc(test_type: TestType) -> TestDesc {
361 name: StaticTestName("whatever"),
363 should_panic: ShouldPanic::No,
371 fn test_exec_time(millis: u64) -> TestExecTime {
372 TestExecTime(Duration::from_millis(millis))
376 fn test_time_options_threshold() {
377 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
378 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
379 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
381 let options = TestTimeOptions {
382 error_on_excess: false,
384 unit_threshold: unit.clone(),
385 integration_threshold: integration.clone(),
386 doctest_threshold: doc.clone(),
390 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
391 (TestType::UnitTest, unit.warn.as_millis(), true, false),
392 (TestType::UnitTest, unit.critical.as_millis(), true, true),
393 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
394 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
395 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
396 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
397 (TestType::DocTest, doc.warn.as_millis(), true, false),
398 (TestType::DocTest, doc.critical.as_millis(), true, true),
401 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
402 let test_desc = typed_test_desc(*test_type);
403 let exec_time = test_exec_time(*time as u64);
405 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
406 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
411 fn parse_ignored_flag() {
412 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
413 let opts = parse_opts(&args).unwrap().unwrap();
414 assert_eq!(opts.run_ignored, RunIgnored::Only);
418 fn parse_show_output_flag() {
419 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
420 let opts = parse_opts(&args).unwrap().unwrap();
421 assert!(opts.options.display_output);
425 fn parse_include_ignored_flag() {
426 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
427 let opts = parse_opts(&args).unwrap().unwrap();
428 assert_eq!(opts.run_ignored, RunIgnored::Yes);
432 pub fn filter_for_ignored_option() {
433 // When we run ignored tests the test filter should filter out all the
434 // unignored tests and flip the ignore flag on the rest to false
436 let mut opts = TestOpts::new();
437 opts.run_tests = true;
438 opts.run_ignored = RunIgnored::Only;
440 let tests = one_ignored_one_unignored_test();
441 let filtered = filter_tests(&opts, tests);
443 assert_eq!(filtered.len(), 1);
444 assert_eq!(filtered[0].desc.name.to_string(), "1");
445 assert!(!filtered[0].desc.ignore);
449 pub fn run_include_ignored_option() {
450 // When we "--include-ignored" tests, the ignore flag should be set to false on
451 // all tests and no test filtered out
453 let mut opts = TestOpts::new();
454 opts.run_tests = true;
455 opts.run_ignored = RunIgnored::Yes;
457 let tests = one_ignored_one_unignored_test();
458 let filtered = filter_tests(&opts, tests);
460 assert_eq!(filtered.len(), 2);
461 assert!(!filtered[0].desc.ignore);
462 assert!(!filtered[1].desc.ignore);
466 pub fn exclude_should_panic_option() {
467 let mut opts = TestOpts::new();
468 opts.run_tests = true;
469 opts.exclude_should_panic = true;
471 let mut tests = one_ignored_one_unignored_test();
472 tests.push(TestDescAndFn {
474 name: StaticTestName("3"),
476 should_panic: ShouldPanic::Yes,
480 test_type: TestType::Unknown,
482 testfn: DynTestFn(Box::new(move || {})),
485 let filtered = filter_tests(&opts, tests);
487 assert_eq!(filtered.len(), 2);
488 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
492 pub fn exact_filter_match() {
493 fn tests() -> Vec<TestDescAndFn> {
494 vec!["base", "base::test", "base::test1", "base::test2"]
496 .map(|name| TestDescAndFn {
498 name: StaticTestName(name),
500 should_panic: ShouldPanic::No,
504 test_type: TestType::Unknown,
506 testfn: DynTestFn(Box::new(move || {})),
512 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
513 assert_eq!(substr.len(), 4);
516 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
517 assert_eq!(substr.len(), 4);
520 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
521 assert_eq!(substr.len(), 3);
524 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
525 assert_eq!(substr.len(), 3);
527 let substr = filter_tests(
528 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
531 assert_eq!(substr.len(), 2);
533 let exact = filter_tests(
534 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
537 assert_eq!(exact.len(), 1);
539 let exact = filter_tests(
540 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
543 assert_eq!(exact.len(), 0);
545 let exact = filter_tests(
546 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
549 assert_eq!(exact.len(), 0);
551 let exact = filter_tests(
552 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
555 assert_eq!(exact.len(), 1);
557 let exact = filter_tests(
559 filters: vec!["base".into(), "base::test".into()],
565 assert_eq!(exact.len(), 2);
569 pub fn sort_tests() {
570 let mut opts = TestOpts::new();
571 opts.run_tests = true;
574 "sha1::test".to_string(),
575 "isize::test_to_str".to_string(),
576 "isize::test_pow".to_string(),
577 "test::do_not_run_ignored_tests".to_string(),
578 "test::ignored_tests_result_in_ignored".to_string(),
579 "test::first_free_arg_should_be_a_filter".to_string(),
580 "test::parse_ignored_flag".to_string(),
581 "test::parse_include_ignored_flag".to_string(),
582 "test::filter_for_ignored_option".to_string(),
583 "test::run_include_ignored_option".to_string(),
584 "test::sort_tests".to_string(),
588 let mut tests = Vec::new();
590 let test = TestDescAndFn {
592 name: DynTestName((*name).clone()),
594 should_panic: ShouldPanic::No,
598 test_type: TestType::Unknown,
600 testfn: DynTestFn(Box::new(testfn)),
606 let filtered = filter_tests(&opts, tests);
609 "isize::test_pow".to_string(),
610 "isize::test_to_str".to_string(),
611 "sha1::test".to_string(),
612 "test::do_not_run_ignored_tests".to_string(),
613 "test::filter_for_ignored_option".to_string(),
614 "test::first_free_arg_should_be_a_filter".to_string(),
615 "test::ignored_tests_result_in_ignored".to_string(),
616 "test::parse_ignored_flag".to_string(),
617 "test::parse_include_ignored_flag".to_string(),
618 "test::run_include_ignored_option".to_string(),
619 "test::sort_tests".to_string(),
622 for (a, b) in expected.iter().zip(filtered) {
623 assert_eq!(*a, b.desc.name.to_string());
628 pub fn test_metricmap_compare() {
629 let mut m1 = MetricMap::new();
630 let mut m2 = MetricMap::new();
631 m1.insert_metric("in-both-noise", 1000.0, 200.0);
632 m2.insert_metric("in-both-noise", 1100.0, 200.0);
634 m1.insert_metric("in-first-noise", 1000.0, 2.0);
635 m2.insert_metric("in-second-noise", 1000.0, 2.0);
637 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
638 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
640 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
641 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
643 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
644 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
646 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
647 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
651 pub fn test_bench_once_no_iter() {
652 fn f(_: &mut Bencher) {}
657 pub fn test_bench_once_iter() {
658 fn f(b: &mut Bencher) {
665 pub fn test_bench_no_iter() {
666 fn f(_: &mut Bencher) {}
668 let (tx, rx) = channel();
670 let desc = TestDesc {
671 name: StaticTestName("f"),
673 should_panic: ShouldPanic::No,
677 test_type: TestType::Unknown,
680 crate::bench::benchmark(TestId(0), desc, tx, true, f);
685 pub fn test_bench_iter() {
686 fn f(b: &mut Bencher) {
690 let (tx, rx) = channel();
692 let desc = TestDesc {
693 name: StaticTestName("f"),
695 should_panic: ShouldPanic::No,
699 test_type: TestType::Unknown,
702 crate::bench::benchmark(TestId(0), desc, tx, true, f);
707 fn should_sort_failures_before_printing_them() {
708 let test_a = TestDesc {
709 name: StaticTestName("a"),
711 should_panic: ShouldPanic::No,
715 test_type: TestType::Unknown,
718 let test_b = TestDesc {
719 name: StaticTestName("b"),
721 should_panic: ShouldPanic::No,
725 test_type: TestType::Unknown,
728 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
730 let st = console::ConsoleTestState {
740 metrics: MetricMap::new(),
741 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
742 options: Options::new(),
743 not_failures: Vec::new(),
744 time_failures: Vec::new(),
747 out.write_failures(&st).unwrap();
748 let s = match out.output_location() {
749 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
750 &OutputLocation::Pretty(_) => unreachable!(),
753 let apos = s.find("a").unwrap();
754 let bpos = s.find("b").unwrap();
755 assert!(apos < bpos);