5 console::OutputLocation,
7 time::{TimeThreshold, TestTimeOptions},
8 formatters::PrettyFormatter,
10 filter_tests, parse_opts, run_test, DynTestFn, DynTestName, MetricMap,
11 RunIgnored, RunStrategy, ShouldPanic, StaticTestName, TestDesc,
12 TestDescAndFn, TestOpts, TrIgnored, TrOk,
13 // FIXME (introduced by #65251)
14 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
15 // TestType, TrFailedMsg, TrIgnored, TrOk,
19 use std::sync::mpsc::channel;
20 use std::time::Duration;
23 fn new() -> TestOpts {
28 force_run_in_process: false,
29 exclude_should_panic: false,
30 run_ignored: RunIgnored::No,
32 bench_benchmarks: false,
36 format: OutputFormat::Pretty,
40 options: Options::new(),
45 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
49 name: StaticTestName("1"),
51 should_panic: ShouldPanic::No,
53 test_type: TestType::Unknown,
55 testfn: DynTestFn(Box::new(move || {})),
59 name: StaticTestName("2"),
61 should_panic: ShouldPanic::No,
63 test_type: TestType::Unknown,
65 testfn: DynTestFn(Box::new(move || {})),
71 pub fn do_not_run_ignored_tests() {
75 let desc = TestDescAndFn {
77 name: StaticTestName("whatever"),
79 should_panic: ShouldPanic::No,
81 test_type: TestType::Unknown,
83 testfn: DynTestFn(Box::new(f)),
85 let (tx, rx) = channel();
86 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
87 let result = rx.recv().unwrap().result;
88 assert_ne!(result, TrOk);
92 pub fn ignored_tests_result_in_ignored() {
94 let desc = TestDescAndFn {
96 name: StaticTestName("whatever"),
98 should_panic: ShouldPanic::No,
100 test_type: TestType::Unknown,
102 testfn: DynTestFn(Box::new(f)),
104 let (tx, rx) = channel();
105 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
106 let result = rx.recv().unwrap().result;
107 assert_eq!(result, TrIgnored);
110 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
112 #[cfg(not(target_os = "emscripten"))]
113 fn test_should_panic() {
117 let desc = TestDescAndFn {
119 name: StaticTestName("whatever"),
121 should_panic: ShouldPanic::Yes,
123 test_type: TestType::Unknown,
125 testfn: DynTestFn(Box::new(f)),
127 let (tx, rx) = channel();
128 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
129 let result = rx.recv().unwrap().result;
130 assert_eq!(result, TrOk);
133 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
135 #[cfg(not(target_os = "emscripten"))]
136 fn test_should_panic_good_message() {
138 panic!("an error message");
140 let desc = TestDescAndFn {
142 name: StaticTestName("whatever"),
144 should_panic: ShouldPanic::YesWithMessage("error message"),
146 test_type: TestType::Unknown,
148 testfn: DynTestFn(Box::new(f)),
150 let (tx, rx) = channel();
151 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
152 let result = rx.recv().unwrap().result;
153 assert_eq!(result, TrOk);
156 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
158 #[cfg(not(target_os = "emscripten"))]
159 fn test_should_panic_bad_message() {
160 use crate::tests::TrFailedMsg;
162 panic!("an error message");
164 let expected = "foobar";
165 let failed_msg = r#"panic did not contain expected string
166 panic message: `"an error message"`,
167 expected substring: `"foobar"`"#;
168 let desc = TestDescAndFn {
170 name: StaticTestName("whatever"),
172 should_panic: ShouldPanic::YesWithMessage(expected),
174 test_type: TestType::Unknown,
176 testfn: DynTestFn(Box::new(f)),
178 let (tx, rx) = channel();
179 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
180 let result = rx.recv().unwrap().result;
181 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
184 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
186 #[cfg(not(target_os = "emscripten"))]
187 fn test_should_panic_non_string_message_type() {
188 use crate::tests::TrFailedMsg;
192 let expected = "foobar";
193 let failed_msg = format!(r#"expected panic with string value,
194 found non-string value: `{:?}`
195 expected substring: `"foobar"`"#, TypeId::of::<i32>());
196 let desc = TestDescAndFn {
198 name: StaticTestName("whatever"),
200 should_panic: ShouldPanic::YesWithMessage(expected),
202 test_type: TestType::Unknown,
204 testfn: DynTestFn(Box::new(f)),
206 let (tx, rx) = channel();
207 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
208 let result = rx.recv().unwrap().result;
209 assert_eq!(result, TrFailedMsg(failed_msg));
212 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
214 #[cfg(not(target_os = "emscripten"))]
215 fn test_should_panic_but_succeeds() {
217 let desc = TestDescAndFn {
219 name: StaticTestName("whatever"),
221 should_panic: ShouldPanic::Yes,
223 test_type: TestType::Unknown,
225 testfn: DynTestFn(Box::new(f)),
227 let (tx, rx) = channel();
228 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
229 let result = rx.recv().unwrap().result;
230 assert_eq!(result, TrFailedMsg("test did not panic as expected".to_string()));
233 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
235 let desc = TestDescAndFn {
237 name: StaticTestName("whatever"),
239 should_panic: ShouldPanic::No,
241 test_type: TestType::Unknown,
243 testfn: DynTestFn(Box::new(f)),
245 let time_options = if report_time {
246 Some(TestTimeOptions::default())
251 let test_opts = TestOpts {
255 let (tx, rx) = channel();
256 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
257 let exec_time = rx.recv().unwrap().exec_time;
262 fn test_should_not_report_time() {
263 let exec_time = report_time_test_template(false);
264 assert!(exec_time.is_none());
268 fn test_should_report_time() {
269 let exec_time = report_time_test_template(true);
270 assert!(exec_time.is_some());
273 fn time_test_failure_template(test_type: TestType) -> TestResult {
275 let desc = TestDescAndFn {
277 name: StaticTestName("whatever"),
279 should_panic: ShouldPanic::No,
283 testfn: DynTestFn(Box::new(f)),
285 // `Default` will initialize all the thresholds to 0 milliseconds.
286 let mut time_options = TestTimeOptions::default();
287 time_options.error_on_excess = true;
289 let test_opts = TestOpts {
290 time_options: Some(time_options),
293 let (tx, rx) = channel();
294 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
295 let result = rx.recv().unwrap().result;
301 fn test_error_on_exceed() {
302 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
304 for test_type in types.iter() {
305 let result = time_test_failure_template(*test_type);
307 assert_eq!(result, TestResult::TrTimedFail);
310 // Check that for unknown tests thresholds aren't applied.
311 let result = time_test_failure_template(TestType::Unknown);
312 assert_eq!(result, TestResult::TrOk);
315 fn typed_test_desc(test_type: TestType) -> TestDesc {
317 name: StaticTestName("whatever"),
319 should_panic: ShouldPanic::No,
325 fn test_exec_time(millis: u64) -> TestExecTime {
326 TestExecTime(Duration::from_millis(millis))
330 fn test_time_options_threshold() {
331 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
332 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
333 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
335 let options = TestTimeOptions {
336 error_on_excess: false,
338 unit_threshold: unit.clone(),
339 integration_threshold: integration.clone(),
340 doctest_threshold: doc.clone(),
344 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
345 (TestType::UnitTest, unit.warn.as_millis(), true, false),
346 (TestType::UnitTest, unit.critical.as_millis(), true, true),
347 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
348 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
349 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
350 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
351 (TestType::DocTest, doc.warn.as_millis(), true, false),
352 (TestType::DocTest, doc.critical.as_millis(), true, true),
355 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
356 let test_desc = typed_test_desc(*test_type);
357 let exec_time = test_exec_time(*time as u64);
359 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
360 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
365 fn parse_ignored_flag() {
367 "progname".to_string(),
368 "filter".to_string(),
369 "--ignored".to_string(),
371 let opts = parse_opts(&args).unwrap().unwrap();
372 assert_eq!(opts.run_ignored, RunIgnored::Only);
376 fn parse_show_output_flag() {
378 "progname".to_string(),
379 "filter".to_string(),
380 "--show-output".to_string(),
382 let opts = parse_opts(&args).unwrap().unwrap();
383 assert!(opts.options.display_output);
387 fn parse_include_ignored_flag() {
389 "progname".to_string(),
390 "filter".to_string(),
391 "-Zunstable-options".to_string(),
392 "--include-ignored".to_string(),
394 let opts = parse_opts(&args).unwrap().unwrap();
395 assert_eq!(opts.run_ignored, RunIgnored::Yes);
399 pub fn filter_for_ignored_option() {
400 // When we run ignored tests the test filter should filter out all the
401 // unignored tests and flip the ignore flag on the rest to false
403 let mut opts = TestOpts::new();
404 opts.run_tests = true;
405 opts.run_ignored = RunIgnored::Only;
407 let tests = one_ignored_one_unignored_test();
408 let filtered = filter_tests(&opts, tests);
410 assert_eq!(filtered.len(), 1);
411 assert_eq!(filtered[0].desc.name.to_string(), "1");
412 assert!(!filtered[0].desc.ignore);
416 pub fn run_include_ignored_option() {
417 // When we "--include-ignored" tests, the ignore flag should be set to false on
418 // all tests and no test filtered out
420 let mut opts = TestOpts::new();
421 opts.run_tests = true;
422 opts.run_ignored = RunIgnored::Yes;
424 let tests = one_ignored_one_unignored_test();
425 let filtered = filter_tests(&opts, tests);
427 assert_eq!(filtered.len(), 2);
428 assert!(!filtered[0].desc.ignore);
429 assert!(!filtered[1].desc.ignore);
433 pub fn exclude_should_panic_option() {
434 let mut opts = TestOpts::new();
435 opts.run_tests = true;
436 opts.exclude_should_panic = true;
438 let mut tests = one_ignored_one_unignored_test();
439 tests.push(TestDescAndFn {
441 name: StaticTestName("3"),
443 should_panic: ShouldPanic::Yes,
445 test_type: TestType::Unknown,
447 testfn: DynTestFn(Box::new(move || {})),
450 let filtered = filter_tests(&opts, tests);
452 assert_eq!(filtered.len(), 2);
453 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
457 pub fn exact_filter_match() {
458 fn tests() -> Vec<TestDescAndFn> {
459 vec!["base", "base::test", "base::test1", "base::test2"]
461 .map(|name| TestDescAndFn {
463 name: StaticTestName(name),
465 should_panic: ShouldPanic::No,
467 test_type: TestType::Unknown,
469 testfn: DynTestFn(Box::new(move || {})),
474 let substr = filter_tests(
476 filter: Some("base".into()),
481 assert_eq!(substr.len(), 4);
483 let substr = filter_tests(
485 filter: Some("bas".into()),
490 assert_eq!(substr.len(), 4);
492 let substr = filter_tests(
494 filter: Some("::test".into()),
499 assert_eq!(substr.len(), 3);
501 let substr = filter_tests(
503 filter: Some("base::test".into()),
508 assert_eq!(substr.len(), 3);
510 let exact = filter_tests(
512 filter: Some("base".into()),
518 assert_eq!(exact.len(), 1);
520 let exact = filter_tests(
522 filter: Some("bas".into()),
528 assert_eq!(exact.len(), 0);
530 let exact = filter_tests(
532 filter: Some("::test".into()),
538 assert_eq!(exact.len(), 0);
540 let exact = filter_tests(
542 filter: Some("base::test".into()),
548 assert_eq!(exact.len(), 1);
552 pub fn sort_tests() {
553 let mut opts = TestOpts::new();
554 opts.run_tests = true;
557 "sha1::test".to_string(),
558 "isize::test_to_str".to_string(),
559 "isize::test_pow".to_string(),
560 "test::do_not_run_ignored_tests".to_string(),
561 "test::ignored_tests_result_in_ignored".to_string(),
562 "test::first_free_arg_should_be_a_filter".to_string(),
563 "test::parse_ignored_flag".to_string(),
564 "test::parse_include_ignored_flag".to_string(),
565 "test::filter_for_ignored_option".to_string(),
566 "test::run_include_ignored_option".to_string(),
567 "test::sort_tests".to_string(),
571 let mut tests = Vec::new();
573 let test = TestDescAndFn {
575 name: DynTestName((*name).clone()),
577 should_panic: ShouldPanic::No,
579 test_type: TestType::Unknown,
581 testfn: DynTestFn(Box::new(testfn)),
587 let filtered = filter_tests(&opts, tests);
590 "isize::test_pow".to_string(),
591 "isize::test_to_str".to_string(),
592 "sha1::test".to_string(),
593 "test::do_not_run_ignored_tests".to_string(),
594 "test::filter_for_ignored_option".to_string(),
595 "test::first_free_arg_should_be_a_filter".to_string(),
596 "test::ignored_tests_result_in_ignored".to_string(),
597 "test::parse_ignored_flag".to_string(),
598 "test::parse_include_ignored_flag".to_string(),
599 "test::run_include_ignored_option".to_string(),
600 "test::sort_tests".to_string(),
603 for (a, b) in expected.iter().zip(filtered) {
604 assert_eq!(*a, b.desc.name.to_string());
609 pub fn test_metricmap_compare() {
610 let mut m1 = MetricMap::new();
611 let mut m2 = MetricMap::new();
612 m1.insert_metric("in-both-noise", 1000.0, 200.0);
613 m2.insert_metric("in-both-noise", 1100.0, 200.0);
615 m1.insert_metric("in-first-noise", 1000.0, 2.0);
616 m2.insert_metric("in-second-noise", 1000.0, 2.0);
618 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
619 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
621 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
622 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
624 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
625 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
627 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
628 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
632 pub fn test_bench_once_no_iter() {
633 fn f(_: &mut Bencher) {}
638 pub fn test_bench_once_iter() {
639 fn f(b: &mut Bencher) {
646 pub fn test_bench_no_iter() {
647 fn f(_: &mut Bencher) {}
649 let (tx, rx) = channel();
651 let desc = TestDesc {
652 name: StaticTestName("f"),
654 should_panic: ShouldPanic::No,
656 test_type: TestType::Unknown,
659 crate::bench::benchmark(desc, tx, true, f);
664 pub fn test_bench_iter() {
665 fn f(b: &mut Bencher) {
669 let (tx, rx) = channel();
671 let desc = TestDesc {
672 name: StaticTestName("f"),
674 should_panic: ShouldPanic::No,
676 test_type: TestType::Unknown,
679 crate::bench::benchmark(desc, tx, true, f);
684 fn should_sort_failures_before_printing_them() {
685 let test_a = TestDesc {
686 name: StaticTestName("a"),
688 should_panic: ShouldPanic::No,
690 test_type: TestType::Unknown,
693 let test_b = TestDesc {
694 name: StaticTestName("b"),
696 should_panic: ShouldPanic::No,
698 test_type: TestType::Unknown,
701 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
703 let st = console::ConsoleTestState {
712 metrics: MetricMap::new(),
713 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
714 options: Options::new(),
715 not_failures: Vec::new(),
716 time_failures: Vec::new(),
719 out.write_failures(&st).unwrap();
720 let s = match out.output_location() {
721 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
722 &OutputLocation::Pretty(_) => unreachable!(),
725 let apos = s.find("a").unwrap();
726 let bpos = s.find("b").unwrap();
727 assert!(apos < bpos);