5 console::OutputLocation,
6 formatters::PrettyFormatter,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
28 time::{TestTimeOptions, TimeThreshold},
31 use std::sync::mpsc::channel;
32 use std::time::Duration;
35 fn new() -> TestOpts {
40 force_run_in_process: false,
41 exclude_should_panic: false,
42 run_ignored: RunIgnored::No,
44 bench_benchmarks: false,
48 format: OutputFormat::Pretty,
52 options: Options::new(),
57 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
61 name: StaticTestName("1"),
63 should_panic: ShouldPanic::No,
65 test_type: TestType::Unknown,
67 testfn: DynTestFn(Box::new(move || {})),
71 name: StaticTestName("2"),
73 should_panic: ShouldPanic::No,
75 test_type: TestType::Unknown,
77 testfn: DynTestFn(Box::new(move || {})),
83 pub fn do_not_run_ignored_tests() {
87 let desc = TestDescAndFn {
89 name: StaticTestName("whatever"),
91 should_panic: ShouldPanic::No,
93 test_type: TestType::Unknown,
95 testfn: DynTestFn(Box::new(f)),
97 let (tx, rx) = channel();
98 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
99 let result = rx.recv().unwrap().result;
100 assert_ne!(result, TrOk);
104 pub fn ignored_tests_result_in_ignored() {
106 let desc = TestDescAndFn {
108 name: StaticTestName("whatever"),
110 should_panic: ShouldPanic::No,
112 test_type: TestType::Unknown,
114 testfn: DynTestFn(Box::new(f)),
116 let (tx, rx) = channel();
117 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
118 let result = rx.recv().unwrap().result;
119 assert_eq!(result, TrIgnored);
122 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
124 #[cfg(not(target_os = "emscripten"))]
125 fn test_should_panic() {
129 let desc = TestDescAndFn {
131 name: StaticTestName("whatever"),
133 should_panic: ShouldPanic::Yes,
135 test_type: TestType::Unknown,
137 testfn: DynTestFn(Box::new(f)),
139 let (tx, rx) = channel();
140 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
141 let result = rx.recv().unwrap().result;
142 assert_eq!(result, TrOk);
145 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
147 #[cfg(not(target_os = "emscripten"))]
148 fn test_should_panic_good_message() {
150 panic!("an error message");
152 let desc = TestDescAndFn {
154 name: StaticTestName("whatever"),
156 should_panic: ShouldPanic::YesWithMessage("error message"),
158 test_type: TestType::Unknown,
160 testfn: DynTestFn(Box::new(f)),
162 let (tx, rx) = channel();
163 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
164 let result = rx.recv().unwrap().result;
165 assert_eq!(result, TrOk);
168 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
170 #[cfg(not(target_os = "emscripten"))]
171 fn test_should_panic_bad_message() {
172 use crate::tests::TrFailedMsg;
174 panic!("an error message");
176 let expected = "foobar";
177 let failed_msg = r#"panic did not contain expected string
178 panic message: `"an error message"`,
179 expected substring: `"foobar"`"#;
180 let desc = TestDescAndFn {
182 name: StaticTestName("whatever"),
184 should_panic: ShouldPanic::YesWithMessage(expected),
186 test_type: TestType::Unknown,
188 testfn: DynTestFn(Box::new(f)),
190 let (tx, rx) = channel();
191 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
192 let result = rx.recv().unwrap().result;
193 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
196 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
198 #[cfg(not(target_os = "emscripten"))]
199 fn test_should_panic_non_string_message_type() {
200 use crate::tests::TrFailedMsg;
204 let expected = "foobar";
205 let failed_msg = format!(
206 r#"expected panic with string value,
207 found non-string value: `{:?}`
208 expected substring: `"foobar"`"#,
211 let desc = TestDescAndFn {
213 name: StaticTestName("whatever"),
215 should_panic: ShouldPanic::YesWithMessage(expected),
217 test_type: TestType::Unknown,
219 testfn: DynTestFn(Box::new(f)),
221 let (tx, rx) = channel();
222 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
223 let result = rx.recv().unwrap().result;
224 assert_eq!(result, TrFailedMsg(failed_msg));
227 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
229 #[cfg(not(target_os = "emscripten"))]
230 fn test_should_panic_but_succeeds() {
232 let desc = TestDescAndFn {
234 name: StaticTestName("whatever"),
236 should_panic: ShouldPanic::Yes,
238 test_type: TestType::Unknown,
240 testfn: DynTestFn(Box::new(f)),
242 let (tx, rx) = channel();
243 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
244 let result = rx.recv().unwrap().result;
245 assert_eq!(result, TrFailedMsg("test did not panic as expected".to_string()));
248 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
250 let desc = TestDescAndFn {
252 name: StaticTestName("whatever"),
254 should_panic: ShouldPanic::No,
256 test_type: TestType::Unknown,
258 testfn: DynTestFn(Box::new(f)),
260 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
262 let test_opts = TestOpts { time_options, ..TestOpts::new() };
263 let (tx, rx) = channel();
264 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
265 let exec_time = rx.recv().unwrap().exec_time;
270 fn test_should_not_report_time() {
271 let exec_time = report_time_test_template(false);
272 assert!(exec_time.is_none());
276 fn test_should_report_time() {
277 let exec_time = report_time_test_template(true);
278 assert!(exec_time.is_some());
281 fn time_test_failure_template(test_type: TestType) -> TestResult {
283 let desc = TestDescAndFn {
285 name: StaticTestName("whatever"),
287 should_panic: ShouldPanic::No,
291 testfn: DynTestFn(Box::new(f)),
293 // `Default` will initialize all the thresholds to 0 milliseconds.
294 let mut time_options = TestTimeOptions::default();
295 time_options.error_on_excess = true;
297 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
298 let (tx, rx) = channel();
299 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
300 let result = rx.recv().unwrap().result;
306 fn test_error_on_exceed() {
307 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
309 for test_type in types.iter() {
310 let result = time_test_failure_template(*test_type);
312 assert_eq!(result, TestResult::TrTimedFail);
315 // Check that for unknown tests thresholds aren't applied.
316 let result = time_test_failure_template(TestType::Unknown);
317 assert_eq!(result, TestResult::TrOk);
320 fn typed_test_desc(test_type: TestType) -> TestDesc {
322 name: StaticTestName("whatever"),
324 should_panic: ShouldPanic::No,
330 fn test_exec_time(millis: u64) -> TestExecTime {
331 TestExecTime(Duration::from_millis(millis))
335 fn test_time_options_threshold() {
336 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
337 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
338 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
340 let options = TestTimeOptions {
341 error_on_excess: false,
343 unit_threshold: unit.clone(),
344 integration_threshold: integration.clone(),
345 doctest_threshold: doc.clone(),
349 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
350 (TestType::UnitTest, unit.warn.as_millis(), true, false),
351 (TestType::UnitTest, unit.critical.as_millis(), true, true),
352 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
353 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
354 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
355 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
356 (TestType::DocTest, doc.warn.as_millis(), true, false),
357 (TestType::DocTest, doc.critical.as_millis(), true, true),
360 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
361 let test_desc = typed_test_desc(*test_type);
362 let exec_time = test_exec_time(*time as u64);
364 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
365 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
370 fn parse_ignored_flag() {
371 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
372 let opts = parse_opts(&args).unwrap().unwrap();
373 assert_eq!(opts.run_ignored, RunIgnored::Only);
377 fn parse_show_output_flag() {
378 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
379 let opts = parse_opts(&args).unwrap().unwrap();
380 assert!(opts.options.display_output);
384 fn parse_include_ignored_flag() {
386 "progname".to_string(),
387 "filter".to_string(),
388 "-Zunstable-options".to_string(),
389 "--include-ignored".to_string(),
391 let opts = parse_opts(&args).unwrap().unwrap();
392 assert_eq!(opts.run_ignored, RunIgnored::Yes);
396 pub fn filter_for_ignored_option() {
397 // When we run ignored tests the test filter should filter out all the
398 // unignored tests and flip the ignore flag on the rest to false
400 let mut opts = TestOpts::new();
401 opts.run_tests = true;
402 opts.run_ignored = RunIgnored::Only;
404 let tests = one_ignored_one_unignored_test();
405 let filtered = filter_tests(&opts, tests);
407 assert_eq!(filtered.len(), 1);
408 assert_eq!(filtered[0].desc.name.to_string(), "1");
409 assert!(!filtered[0].desc.ignore);
413 pub fn run_include_ignored_option() {
414 // When we "--include-ignored" tests, the ignore flag should be set to false on
415 // all tests and no test filtered out
417 let mut opts = TestOpts::new();
418 opts.run_tests = true;
419 opts.run_ignored = RunIgnored::Yes;
421 let tests = one_ignored_one_unignored_test();
422 let filtered = filter_tests(&opts, tests);
424 assert_eq!(filtered.len(), 2);
425 assert!(!filtered[0].desc.ignore);
426 assert!(!filtered[1].desc.ignore);
430 pub fn exclude_should_panic_option() {
431 let mut opts = TestOpts::new();
432 opts.run_tests = true;
433 opts.exclude_should_panic = true;
435 let mut tests = one_ignored_one_unignored_test();
436 tests.push(TestDescAndFn {
438 name: StaticTestName("3"),
440 should_panic: ShouldPanic::Yes,
442 test_type: TestType::Unknown,
444 testfn: DynTestFn(Box::new(move || {})),
447 let filtered = filter_tests(&opts, tests);
449 assert_eq!(filtered.len(), 2);
450 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
454 pub fn exact_filter_match() {
455 fn tests() -> Vec<TestDescAndFn> {
456 vec!["base", "base::test", "base::test1", "base::test2"]
458 .map(|name| TestDescAndFn {
460 name: StaticTestName(name),
462 should_panic: ShouldPanic::No,
464 test_type: TestType::Unknown,
466 testfn: DynTestFn(Box::new(move || {})),
472 filter_tests(&TestOpts { filter: Some("base".into()), ..TestOpts::new() }, tests());
473 assert_eq!(substr.len(), 4);
475 let substr = filter_tests(&TestOpts { filter: Some("bas".into()), ..TestOpts::new() }, tests());
476 assert_eq!(substr.len(), 4);
479 filter_tests(&TestOpts { filter: Some("::test".into()), ..TestOpts::new() }, tests());
480 assert_eq!(substr.len(), 3);
483 filter_tests(&TestOpts { filter: Some("base::test".into()), ..TestOpts::new() }, tests());
484 assert_eq!(substr.len(), 3);
486 let exact = filter_tests(
487 &TestOpts { filter: Some("base".into()), filter_exact: true, ..TestOpts::new() },
490 assert_eq!(exact.len(), 1);
492 let exact = filter_tests(
493 &TestOpts { filter: Some("bas".into()), filter_exact: true, ..TestOpts::new() },
496 assert_eq!(exact.len(), 0);
498 let exact = filter_tests(
499 &TestOpts { filter: Some("::test".into()), filter_exact: true, ..TestOpts::new() },
502 assert_eq!(exact.len(), 0);
504 let exact = filter_tests(
505 &TestOpts { filter: Some("base::test".into()), filter_exact: true, ..TestOpts::new() },
508 assert_eq!(exact.len(), 1);
512 pub fn sort_tests() {
513 let mut opts = TestOpts::new();
514 opts.run_tests = true;
517 "sha1::test".to_string(),
518 "isize::test_to_str".to_string(),
519 "isize::test_pow".to_string(),
520 "test::do_not_run_ignored_tests".to_string(),
521 "test::ignored_tests_result_in_ignored".to_string(),
522 "test::first_free_arg_should_be_a_filter".to_string(),
523 "test::parse_ignored_flag".to_string(),
524 "test::parse_include_ignored_flag".to_string(),
525 "test::filter_for_ignored_option".to_string(),
526 "test::run_include_ignored_option".to_string(),
527 "test::sort_tests".to_string(),
531 let mut tests = Vec::new();
533 let test = TestDescAndFn {
535 name: DynTestName((*name).clone()),
537 should_panic: ShouldPanic::No,
539 test_type: TestType::Unknown,
541 testfn: DynTestFn(Box::new(testfn)),
547 let filtered = filter_tests(&opts, tests);
550 "isize::test_pow".to_string(),
551 "isize::test_to_str".to_string(),
552 "sha1::test".to_string(),
553 "test::do_not_run_ignored_tests".to_string(),
554 "test::filter_for_ignored_option".to_string(),
555 "test::first_free_arg_should_be_a_filter".to_string(),
556 "test::ignored_tests_result_in_ignored".to_string(),
557 "test::parse_ignored_flag".to_string(),
558 "test::parse_include_ignored_flag".to_string(),
559 "test::run_include_ignored_option".to_string(),
560 "test::sort_tests".to_string(),
563 for (a, b) in expected.iter().zip(filtered) {
564 assert_eq!(*a, b.desc.name.to_string());
569 pub fn test_metricmap_compare() {
570 let mut m1 = MetricMap::new();
571 let mut m2 = MetricMap::new();
572 m1.insert_metric("in-both-noise", 1000.0, 200.0);
573 m2.insert_metric("in-both-noise", 1100.0, 200.0);
575 m1.insert_metric("in-first-noise", 1000.0, 2.0);
576 m2.insert_metric("in-second-noise", 1000.0, 2.0);
578 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
579 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
581 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
582 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
584 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
585 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
587 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
588 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
592 pub fn test_bench_once_no_iter() {
593 fn f(_: &mut Bencher) {}
598 pub fn test_bench_once_iter() {
599 fn f(b: &mut Bencher) {
606 pub fn test_bench_no_iter() {
607 fn f(_: &mut Bencher) {}
609 let (tx, rx) = channel();
611 let desc = TestDesc {
612 name: StaticTestName("f"),
614 should_panic: ShouldPanic::No,
616 test_type: TestType::Unknown,
619 crate::bench::benchmark(desc, tx, true, f);
624 pub fn test_bench_iter() {
625 fn f(b: &mut Bencher) {
629 let (tx, rx) = channel();
631 let desc = TestDesc {
632 name: StaticTestName("f"),
634 should_panic: ShouldPanic::No,
636 test_type: TestType::Unknown,
639 crate::bench::benchmark(desc, tx, true, f);
644 fn should_sort_failures_before_printing_them() {
645 let test_a = TestDesc {
646 name: StaticTestName("a"),
648 should_panic: ShouldPanic::No,
650 test_type: TestType::Unknown,
653 let test_b = TestDesc {
654 name: StaticTestName("b"),
656 should_panic: ShouldPanic::No,
658 test_type: TestType::Unknown,
661 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
663 let st = console::ConsoleTestState {
672 metrics: MetricMap::new(),
673 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
674 options: Options::new(),
675 not_failures: Vec::new(),
676 time_failures: Vec::new(),
679 out.write_failures(&st).unwrap();
680 let s = match out.output_location() {
681 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
682 &OutputLocation::Pretty(_) => unreachable!(),
685 let apos = s.find("a").unwrap();
686 let bpos = s.find("b").unwrap();
687 assert!(apos < bpos);