5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
34 fn new() -> TestOpts {
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
43 bench_benchmarks: false,
47 format: OutputFormat::Pretty,
51 options: Options::new(),
56 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
60 name: StaticTestName("1"),
62 should_panic: ShouldPanic::No,
64 test_type: TestType::Unknown,
66 testfn: DynTestFn(Box::new(move || {})),
70 name: StaticTestName("2"),
72 should_panic: ShouldPanic::No,
74 test_type: TestType::Unknown,
76 testfn: DynTestFn(Box::new(move || {})),
82 pub fn do_not_run_ignored_tests() {
86 let desc = TestDescAndFn {
88 name: StaticTestName("whatever"),
90 should_panic: ShouldPanic::No,
92 test_type: TestType::Unknown,
94 testfn: DynTestFn(Box::new(f)),
96 let (tx, rx) = channel();
97 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
98 let result = rx.recv().unwrap().result;
99 assert_ne!(result, TrOk);
103 pub fn ignored_tests_result_in_ignored() {
105 let desc = TestDescAndFn {
107 name: StaticTestName("whatever"),
109 should_panic: ShouldPanic::No,
111 test_type: TestType::Unknown,
113 testfn: DynTestFn(Box::new(f)),
115 let (tx, rx) = channel();
116 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
117 let result = rx.recv().unwrap().result;
118 assert_eq!(result, TrIgnored);
121 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
123 #[cfg(not(target_os = "emscripten"))]
124 fn test_should_panic() {
128 let desc = TestDescAndFn {
130 name: StaticTestName("whatever"),
132 should_panic: ShouldPanic::Yes,
134 test_type: TestType::Unknown,
136 testfn: DynTestFn(Box::new(f)),
138 let (tx, rx) = channel();
139 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
140 let result = rx.recv().unwrap().result;
141 assert_eq!(result, TrOk);
144 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
146 #[cfg(not(target_os = "emscripten"))]
147 fn test_should_panic_good_message() {
149 panic!("an error message");
151 let desc = TestDescAndFn {
153 name: StaticTestName("whatever"),
155 should_panic: ShouldPanic::YesWithMessage("error message"),
157 test_type: TestType::Unknown,
159 testfn: DynTestFn(Box::new(f)),
161 let (tx, rx) = channel();
162 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
163 let result = rx.recv().unwrap().result;
164 assert_eq!(result, TrOk);
167 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
169 #[cfg(not(target_os = "emscripten"))]
170 fn test_should_panic_bad_message() {
171 use crate::tests::TrFailedMsg;
173 panic!("an error message");
175 let expected = "foobar";
176 let failed_msg = r#"panic did not contain expected string
177 panic message: `"an error message"`,
178 expected substring: `"foobar"`"#;
179 let desc = TestDescAndFn {
181 name: StaticTestName("whatever"),
183 should_panic: ShouldPanic::YesWithMessage(expected),
185 test_type: TestType::Unknown,
187 testfn: DynTestFn(Box::new(f)),
189 let (tx, rx) = channel();
190 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
191 let result = rx.recv().unwrap().result;
192 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
195 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
197 #[cfg(not(target_os = "emscripten"))]
198 fn test_should_panic_non_string_message_type() {
199 use crate::tests::TrFailedMsg;
200 use std::any::TypeId;
202 std::panic::panic_any(1i32);
204 let expected = "foobar";
205 let failed_msg = format!(
206 r#"expected panic with string value,
207 found non-string value: `{:?}`
208 expected substring: `"foobar"`"#,
211 let desc = TestDescAndFn {
213 name: StaticTestName("whatever"),
215 should_panic: ShouldPanic::YesWithMessage(expected),
217 test_type: TestType::Unknown,
219 testfn: DynTestFn(Box::new(f)),
221 let (tx, rx) = channel();
222 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
223 let result = rx.recv().unwrap().result;
224 assert_eq!(result, TrFailedMsg(failed_msg));
227 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
229 #[cfg(not(target_os = "emscripten"))]
230 fn test_should_panic_but_succeeds() {
231 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
233 for &should_panic in should_panic_variants.iter() {
235 let desc = TestDescAndFn {
237 name: StaticTestName("whatever"),
241 test_type: TestType::Unknown,
243 testfn: DynTestFn(Box::new(f)),
245 let (tx, rx) = channel();
246 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
247 let result = rx.recv().unwrap().result;
250 TrFailedMsg("test did not panic as expected".to_string()),
251 "should_panic == {:?}",
257 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
259 let desc = TestDescAndFn {
261 name: StaticTestName("whatever"),
263 should_panic: ShouldPanic::No,
265 test_type: TestType::Unknown,
267 testfn: DynTestFn(Box::new(f)),
269 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
271 let test_opts = TestOpts { time_options, ..TestOpts::new() };
272 let (tx, rx) = channel();
273 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
274 let exec_time = rx.recv().unwrap().exec_time;
279 fn test_should_not_report_time() {
280 let exec_time = report_time_test_template(false);
281 assert!(exec_time.is_none());
285 fn test_should_report_time() {
286 let exec_time = report_time_test_template(true);
287 assert!(exec_time.is_some());
290 fn time_test_failure_template(test_type: TestType) -> TestResult {
292 let desc = TestDescAndFn {
294 name: StaticTestName("whatever"),
296 should_panic: ShouldPanic::No,
300 testfn: DynTestFn(Box::new(f)),
302 // `Default` will initialize all the thresholds to 0 milliseconds.
303 let mut time_options = TestTimeOptions::default();
304 time_options.error_on_excess = true;
306 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
307 let (tx, rx) = channel();
308 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
309 let result = rx.recv().unwrap().result;
315 fn test_error_on_exceed() {
316 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
318 for test_type in types.iter() {
319 let result = time_test_failure_template(*test_type);
321 assert_eq!(result, TestResult::TrTimedFail);
324 // Check that for unknown tests thresholds aren't applied.
325 let result = time_test_failure_template(TestType::Unknown);
326 assert_eq!(result, TestResult::TrOk);
329 fn typed_test_desc(test_type: TestType) -> TestDesc {
331 name: StaticTestName("whatever"),
333 should_panic: ShouldPanic::No,
339 fn test_exec_time(millis: u64) -> TestExecTime {
340 TestExecTime(Duration::from_millis(millis))
344 fn test_time_options_threshold() {
345 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
346 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
347 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
349 let options = TestTimeOptions {
350 error_on_excess: false,
352 unit_threshold: unit.clone(),
353 integration_threshold: integration.clone(),
354 doctest_threshold: doc.clone(),
358 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
359 (TestType::UnitTest, unit.warn.as_millis(), true, false),
360 (TestType::UnitTest, unit.critical.as_millis(), true, true),
361 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
362 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
363 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
364 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
365 (TestType::DocTest, doc.warn.as_millis(), true, false),
366 (TestType::DocTest, doc.critical.as_millis(), true, true),
369 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
370 let test_desc = typed_test_desc(*test_type);
371 let exec_time = test_exec_time(*time as u64);
373 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
374 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
379 fn parse_ignored_flag() {
380 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
381 let opts = parse_opts(&args).unwrap().unwrap();
382 assert_eq!(opts.run_ignored, RunIgnored::Only);
386 fn parse_show_output_flag() {
387 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
388 let opts = parse_opts(&args).unwrap().unwrap();
389 assert!(opts.options.display_output);
393 fn parse_include_ignored_flag() {
394 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
395 let opts = parse_opts(&args).unwrap().unwrap();
396 assert_eq!(opts.run_ignored, RunIgnored::Yes);
400 pub fn filter_for_ignored_option() {
401 // When we run ignored tests the test filter should filter out all the
402 // unignored tests and flip the ignore flag on the rest to false
404 let mut opts = TestOpts::new();
405 opts.run_tests = true;
406 opts.run_ignored = RunIgnored::Only;
408 let tests = one_ignored_one_unignored_test();
409 let filtered = filter_tests(&opts, tests);
411 assert_eq!(filtered.len(), 1);
412 assert_eq!(filtered[0].desc.name.to_string(), "1");
413 assert!(!filtered[0].desc.ignore);
417 pub fn run_include_ignored_option() {
418 // When we "--include-ignored" tests, the ignore flag should be set to false on
419 // all tests and no test filtered out
421 let mut opts = TestOpts::new();
422 opts.run_tests = true;
423 opts.run_ignored = RunIgnored::Yes;
425 let tests = one_ignored_one_unignored_test();
426 let filtered = filter_tests(&opts, tests);
428 assert_eq!(filtered.len(), 2);
429 assert!(!filtered[0].desc.ignore);
430 assert!(!filtered[1].desc.ignore);
434 pub fn exclude_should_panic_option() {
435 let mut opts = TestOpts::new();
436 opts.run_tests = true;
437 opts.exclude_should_panic = true;
439 let mut tests = one_ignored_one_unignored_test();
440 tests.push(TestDescAndFn {
442 name: StaticTestName("3"),
444 should_panic: ShouldPanic::Yes,
446 test_type: TestType::Unknown,
448 testfn: DynTestFn(Box::new(move || {})),
451 let filtered = filter_tests(&opts, tests);
453 assert_eq!(filtered.len(), 2);
454 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
458 pub fn exact_filter_match() {
459 fn tests() -> Vec<TestDescAndFn> {
460 vec!["base", "base::test", "base::test1", "base::test2"]
462 .map(|name| TestDescAndFn {
464 name: StaticTestName(name),
466 should_panic: ShouldPanic::No,
468 test_type: TestType::Unknown,
470 testfn: DynTestFn(Box::new(move || {})),
476 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
477 assert_eq!(substr.len(), 4);
480 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
481 assert_eq!(substr.len(), 4);
484 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
485 assert_eq!(substr.len(), 3);
488 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
489 assert_eq!(substr.len(), 3);
491 let substr = filter_tests(
492 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
495 assert_eq!(substr.len(), 2);
497 let exact = filter_tests(
498 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
501 assert_eq!(exact.len(), 1);
503 let exact = filter_tests(
504 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
507 assert_eq!(exact.len(), 0);
509 let exact = filter_tests(
510 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
513 assert_eq!(exact.len(), 0);
515 let exact = filter_tests(
516 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
519 assert_eq!(exact.len(), 1);
521 let exact = filter_tests(
523 filters: vec!["base".into(), "base::test".into()],
529 assert_eq!(exact.len(), 2);
533 pub fn sort_tests() {
534 let mut opts = TestOpts::new();
535 opts.run_tests = true;
538 "sha1::test".to_string(),
539 "isize::test_to_str".to_string(),
540 "isize::test_pow".to_string(),
541 "test::do_not_run_ignored_tests".to_string(),
542 "test::ignored_tests_result_in_ignored".to_string(),
543 "test::first_free_arg_should_be_a_filter".to_string(),
544 "test::parse_ignored_flag".to_string(),
545 "test::parse_include_ignored_flag".to_string(),
546 "test::filter_for_ignored_option".to_string(),
547 "test::run_include_ignored_option".to_string(),
548 "test::sort_tests".to_string(),
552 let mut tests = Vec::new();
554 let test = TestDescAndFn {
556 name: DynTestName((*name).clone()),
558 should_panic: ShouldPanic::No,
560 test_type: TestType::Unknown,
562 testfn: DynTestFn(Box::new(testfn)),
568 let filtered = filter_tests(&opts, tests);
571 "isize::test_pow".to_string(),
572 "isize::test_to_str".to_string(),
573 "sha1::test".to_string(),
574 "test::do_not_run_ignored_tests".to_string(),
575 "test::filter_for_ignored_option".to_string(),
576 "test::first_free_arg_should_be_a_filter".to_string(),
577 "test::ignored_tests_result_in_ignored".to_string(),
578 "test::parse_ignored_flag".to_string(),
579 "test::parse_include_ignored_flag".to_string(),
580 "test::run_include_ignored_option".to_string(),
581 "test::sort_tests".to_string(),
584 for (a, b) in expected.iter().zip(filtered) {
585 assert_eq!(*a, b.desc.name.to_string());
590 pub fn test_metricmap_compare() {
591 let mut m1 = MetricMap::new();
592 let mut m2 = MetricMap::new();
593 m1.insert_metric("in-both-noise", 1000.0, 200.0);
594 m2.insert_metric("in-both-noise", 1100.0, 200.0);
596 m1.insert_metric("in-first-noise", 1000.0, 2.0);
597 m2.insert_metric("in-second-noise", 1000.0, 2.0);
599 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
600 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
602 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
603 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
605 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
606 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
608 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
609 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
613 pub fn test_bench_once_no_iter() {
614 fn f(_: &mut Bencher) {}
619 pub fn test_bench_once_iter() {
620 fn f(b: &mut Bencher) {
627 pub fn test_bench_no_iter() {
628 fn f(_: &mut Bencher) {}
630 let (tx, rx) = channel();
632 let desc = TestDesc {
633 name: StaticTestName("f"),
635 should_panic: ShouldPanic::No,
637 test_type: TestType::Unknown,
640 crate::bench::benchmark(desc, tx, true, f);
645 pub fn test_bench_iter() {
646 fn f(b: &mut Bencher) {
650 let (tx, rx) = channel();
652 let desc = TestDesc {
653 name: StaticTestName("f"),
655 should_panic: ShouldPanic::No,
657 test_type: TestType::Unknown,
660 crate::bench::benchmark(desc, tx, true, f);
665 fn should_sort_failures_before_printing_them() {
666 let test_a = TestDesc {
667 name: StaticTestName("a"),
669 should_panic: ShouldPanic::No,
671 test_type: TestType::Unknown,
674 let test_b = TestDesc {
675 name: StaticTestName("b"),
677 should_panic: ShouldPanic::No,
679 test_type: TestType::Unknown,
682 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
684 let st = console::ConsoleTestState {
694 metrics: MetricMap::new(),
695 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
696 options: Options::new(),
697 not_failures: Vec::new(),
698 time_failures: Vec::new(),
701 out.write_failures(&st).unwrap();
702 let s = match out.output_location() {
703 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
704 &OutputLocation::Pretty(_) => unreachable!(),
707 let apos = s.find("a").unwrap();
708 let bpos = s.find("b").unwrap();
709 assert!(apos < bpos);