5 console::OutputLocation,
7 time::{TimeThreshold, TestTimeOptions},
8 formatters::PrettyFormatter,
10 filter_tests, parse_opts, run_test, DynTestFn, DynTestName, MetricMap,
11 RunIgnored, RunStrategy, ShouldPanic, StaticTestName, TestDesc,
12 TestDescAndFn, TestOpts, TrIgnored, TrOk,
13 // FIXME (introduced by #65251)
14 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
15 // TestType, TrFailedMsg, TrIgnored, TrOk,
18 use std::sync::mpsc::channel;
19 use std::time::Duration;
22 fn new() -> TestOpts {
27 exclude_should_panic: false,
28 run_ignored: RunIgnored::No,
30 bench_benchmarks: false,
34 format: OutputFormat::Pretty,
38 options: Options::new(),
43 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
47 name: StaticTestName("1"),
49 should_panic: ShouldPanic::No,
51 test_type: TestType::Unknown,
53 testfn: DynTestFn(Box::new(move || {})),
57 name: StaticTestName("2"),
59 should_panic: ShouldPanic::No,
61 test_type: TestType::Unknown,
63 testfn: DynTestFn(Box::new(move || {})),
69 pub fn do_not_run_ignored_tests() {
73 let desc = TestDescAndFn {
75 name: StaticTestName("whatever"),
77 should_panic: ShouldPanic::No,
79 test_type: TestType::Unknown,
81 testfn: DynTestFn(Box::new(f)),
83 let (tx, rx) = channel();
84 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
85 let result = rx.recv().unwrap().result;
86 assert!(result != TrOk);
90 pub fn ignored_tests_result_in_ignored() {
92 let desc = TestDescAndFn {
94 name: StaticTestName("whatever"),
96 should_panic: ShouldPanic::No,
98 test_type: TestType::Unknown,
100 testfn: DynTestFn(Box::new(f)),
102 let (tx, rx) = channel();
103 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
104 let result = rx.recv().unwrap().result;
105 assert!(result == TrIgnored);
108 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
110 #[cfg(not(target_os = "emscripten"))]
111 fn test_should_panic() {
115 let desc = TestDescAndFn {
117 name: StaticTestName("whatever"),
119 should_panic: ShouldPanic::Yes,
121 test_type: TestType::Unknown,
123 testfn: DynTestFn(Box::new(f)),
125 let (tx, rx) = channel();
126 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
127 let result = rx.recv().unwrap().result;
128 assert!(result == TrOk);
131 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
133 #[cfg(not(target_os = "emscripten"))]
134 fn test_should_panic_good_message() {
136 panic!("an error message");
138 let desc = TestDescAndFn {
140 name: StaticTestName("whatever"),
142 should_panic: ShouldPanic::YesWithMessage("error message"),
144 test_type: TestType::Unknown,
146 testfn: DynTestFn(Box::new(f)),
148 let (tx, rx) = channel();
149 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
150 let result = rx.recv().unwrap().result;
151 assert!(result == TrOk);
154 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
156 #[cfg(not(target_os = "emscripten"))]
157 fn test_should_panic_bad_message() {
158 use crate::tests::TrFailedMsg;
160 panic!("an error message");
162 let expected = "foobar";
163 let failed_msg = "panic did not include expected string";
164 let desc = TestDescAndFn {
166 name: StaticTestName("whatever"),
168 should_panic: ShouldPanic::YesWithMessage(expected),
170 test_type: TestType::Unknown,
172 testfn: DynTestFn(Box::new(f)),
174 let (tx, rx) = channel();
175 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
176 let result = rx.recv().unwrap().result;
177 assert!(result == TrFailedMsg(format!("{} '{}'", failed_msg, expected)));
180 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
182 #[cfg(not(target_os = "emscripten"))]
183 fn test_should_panic_but_succeeds() {
185 let desc = TestDescAndFn {
187 name: StaticTestName("whatever"),
189 should_panic: ShouldPanic::Yes,
191 test_type: TestType::Unknown,
193 testfn: DynTestFn(Box::new(f)),
195 let (tx, rx) = channel();
196 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
197 let result = rx.recv().unwrap().result;
198 assert!(result == TrFailedMsg("test did not panic as expected".to_string()));
201 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
203 let desc = TestDescAndFn {
205 name: StaticTestName("whatever"),
207 should_panic: ShouldPanic::No,
209 test_type: TestType::Unknown,
211 testfn: DynTestFn(Box::new(f)),
213 let time_options = if report_time {
214 Some(TestTimeOptions::default())
219 let test_opts = TestOpts {
223 let (tx, rx) = channel();
224 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
225 let exec_time = rx.recv().unwrap().exec_time;
230 fn test_should_not_report_time() {
231 let exec_time = report_time_test_template(false);
232 assert!(exec_time.is_none());
236 fn test_should_report_time() {
237 let exec_time = report_time_test_template(true);
238 assert!(exec_time.is_some());
241 fn time_test_failure_template(test_type: TestType) -> TestResult {
243 let desc = TestDescAndFn {
245 name: StaticTestName("whatever"),
247 should_panic: ShouldPanic::No,
251 testfn: DynTestFn(Box::new(f)),
253 // `Default` will initialize all the thresholds to 0 milliseconds.
254 let mut time_options = TestTimeOptions::default();
255 time_options.error_on_excess = true;
257 let test_opts = TestOpts {
258 time_options: Some(time_options),
261 let (tx, rx) = channel();
262 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
263 let result = rx.recv().unwrap().result;
269 fn test_error_on_exceed() {
270 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
272 for test_type in types.iter() {
273 let result = time_test_failure_template(*test_type);
275 assert_eq!(result, TestResult::TrTimedFail);
278 // Check that for unknown tests thresholds aren't applied.
279 let result = time_test_failure_template(TestType::Unknown);
280 assert_eq!(result, TestResult::TrOk);
283 fn typed_test_desc(test_type: TestType) -> TestDesc {
285 name: StaticTestName("whatever"),
287 should_panic: ShouldPanic::No,
293 fn test_exec_time(millis: u64) -> TestExecTime {
294 TestExecTime(Duration::from_millis(millis))
298 fn test_time_options_threshold() {
299 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
300 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
301 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
303 let options = TestTimeOptions {
304 error_on_excess: false,
306 unit_threshold: unit.clone(),
307 integration_threshold: integration.clone(),
308 doctest_threshold: doc.clone(),
312 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
313 (TestType::UnitTest, unit.warn.as_millis(), true, false),
314 (TestType::UnitTest, unit.critical.as_millis(), true, true),
315 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
316 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
317 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
318 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
319 (TestType::DocTest, doc.warn.as_millis(), true, false),
320 (TestType::DocTest, doc.critical.as_millis(), true, true),
323 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
324 let test_desc = typed_test_desc(*test_type);
325 let exec_time = test_exec_time(*time as u64);
327 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
328 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
333 fn parse_ignored_flag() {
335 "progname".to_string(),
336 "filter".to_string(),
337 "--ignored".to_string(),
339 let opts = parse_opts(&args).unwrap().unwrap();
340 assert_eq!(opts.run_ignored, RunIgnored::Only);
344 fn parse_show_output_flag() {
346 "progname".to_string(),
347 "filter".to_string(),
348 "--show-output".to_string(),
350 let opts = parse_opts(&args).unwrap().unwrap();
351 assert!(opts.options.display_output);
355 fn parse_include_ignored_flag() {
357 "progname".to_string(),
358 "filter".to_string(),
359 "-Zunstable-options".to_string(),
360 "--include-ignored".to_string(),
362 let opts = parse_opts(&args).unwrap().unwrap();
363 assert_eq!(opts.run_ignored, RunIgnored::Yes);
367 pub fn filter_for_ignored_option() {
368 // When we run ignored tests the test filter should filter out all the
369 // unignored tests and flip the ignore flag on the rest to false
371 let mut opts = TestOpts::new();
372 opts.run_tests = true;
373 opts.run_ignored = RunIgnored::Only;
375 let tests = one_ignored_one_unignored_test();
376 let filtered = filter_tests(&opts, tests);
378 assert_eq!(filtered.len(), 1);
379 assert_eq!(filtered[0].desc.name.to_string(), "1");
380 assert!(!filtered[0].desc.ignore);
384 pub fn run_include_ignored_option() {
385 // When we "--include-ignored" tests, the ignore flag should be set to false on
386 // all tests and no test filtered out
388 let mut opts = TestOpts::new();
389 opts.run_tests = true;
390 opts.run_ignored = RunIgnored::Yes;
392 let tests = one_ignored_one_unignored_test();
393 let filtered = filter_tests(&opts, tests);
395 assert_eq!(filtered.len(), 2);
396 assert!(!filtered[0].desc.ignore);
397 assert!(!filtered[1].desc.ignore);
401 pub fn exclude_should_panic_option() {
402 let mut opts = TestOpts::new();
403 opts.run_tests = true;
404 opts.exclude_should_panic = true;
406 let mut tests = one_ignored_one_unignored_test();
407 tests.push(TestDescAndFn {
409 name: StaticTestName("3"),
411 should_panic: ShouldPanic::Yes,
413 test_type: TestType::Unknown,
415 testfn: DynTestFn(Box::new(move || {})),
418 let filtered = filter_tests(&opts, tests);
420 assert_eq!(filtered.len(), 2);
421 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
425 pub fn exact_filter_match() {
426 fn tests() -> Vec<TestDescAndFn> {
427 vec!["base", "base::test", "base::test1", "base::test2"]
429 .map(|name| TestDescAndFn {
431 name: StaticTestName(name),
433 should_panic: ShouldPanic::No,
435 test_type: TestType::Unknown,
437 testfn: DynTestFn(Box::new(move || {})),
442 let substr = filter_tests(
444 filter: Some("base".into()),
449 assert_eq!(substr.len(), 4);
451 let substr = filter_tests(
453 filter: Some("bas".into()),
458 assert_eq!(substr.len(), 4);
460 let substr = filter_tests(
462 filter: Some("::test".into()),
467 assert_eq!(substr.len(), 3);
469 let substr = filter_tests(
471 filter: Some("base::test".into()),
476 assert_eq!(substr.len(), 3);
478 let exact = filter_tests(
480 filter: Some("base".into()),
486 assert_eq!(exact.len(), 1);
488 let exact = filter_tests(
490 filter: Some("bas".into()),
496 assert_eq!(exact.len(), 0);
498 let exact = filter_tests(
500 filter: Some("::test".into()),
506 assert_eq!(exact.len(), 0);
508 let exact = filter_tests(
510 filter: Some("base::test".into()),
516 assert_eq!(exact.len(), 1);
520 pub fn sort_tests() {
521 let mut opts = TestOpts::new();
522 opts.run_tests = true;
525 "sha1::test".to_string(),
526 "isize::test_to_str".to_string(),
527 "isize::test_pow".to_string(),
528 "test::do_not_run_ignored_tests".to_string(),
529 "test::ignored_tests_result_in_ignored".to_string(),
530 "test::first_free_arg_should_be_a_filter".to_string(),
531 "test::parse_ignored_flag".to_string(),
532 "test::parse_include_ignored_flag".to_string(),
533 "test::filter_for_ignored_option".to_string(),
534 "test::run_include_ignored_option".to_string(),
535 "test::sort_tests".to_string(),
539 let mut tests = Vec::new();
541 let test = TestDescAndFn {
543 name: DynTestName((*name).clone()),
545 should_panic: ShouldPanic::No,
547 test_type: TestType::Unknown,
549 testfn: DynTestFn(Box::new(testfn)),
555 let filtered = filter_tests(&opts, tests);
558 "isize::test_pow".to_string(),
559 "isize::test_to_str".to_string(),
560 "sha1::test".to_string(),
561 "test::do_not_run_ignored_tests".to_string(),
562 "test::filter_for_ignored_option".to_string(),
563 "test::first_free_arg_should_be_a_filter".to_string(),
564 "test::ignored_tests_result_in_ignored".to_string(),
565 "test::parse_ignored_flag".to_string(),
566 "test::parse_include_ignored_flag".to_string(),
567 "test::run_include_ignored_option".to_string(),
568 "test::sort_tests".to_string(),
571 for (a, b) in expected.iter().zip(filtered) {
572 assert!(*a == b.desc.name.to_string());
577 pub fn test_metricmap_compare() {
578 let mut m1 = MetricMap::new();
579 let mut m2 = MetricMap::new();
580 m1.insert_metric("in-both-noise", 1000.0, 200.0);
581 m2.insert_metric("in-both-noise", 1100.0, 200.0);
583 m1.insert_metric("in-first-noise", 1000.0, 2.0);
584 m2.insert_metric("in-second-noise", 1000.0, 2.0);
586 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
587 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
589 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
590 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
592 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
593 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
595 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
596 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
600 pub fn test_bench_once_no_iter() {
601 fn f(_: &mut Bencher) {}
606 pub fn test_bench_once_iter() {
607 fn f(b: &mut Bencher) {
614 pub fn test_bench_no_iter() {
615 fn f(_: &mut Bencher) {}
617 let (tx, rx) = channel();
619 let desc = TestDesc {
620 name: StaticTestName("f"),
622 should_panic: ShouldPanic::No,
624 test_type: TestType::Unknown,
627 crate::bench::benchmark(desc, tx, true, f);
632 pub fn test_bench_iter() {
633 fn f(b: &mut Bencher) {
637 let (tx, rx) = channel();
639 let desc = TestDesc {
640 name: StaticTestName("f"),
642 should_panic: ShouldPanic::No,
644 test_type: TestType::Unknown,
647 crate::bench::benchmark(desc, tx, true, f);
652 fn should_sort_failures_before_printing_them() {
653 let test_a = TestDesc {
654 name: StaticTestName("a"),
656 should_panic: ShouldPanic::No,
658 test_type: TestType::Unknown,
661 let test_b = TestDesc {
662 name: StaticTestName("b"),
664 should_panic: ShouldPanic::No,
666 test_type: TestType::Unknown,
669 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
671 let st = console::ConsoleTestState {
680 metrics: MetricMap::new(),
681 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
682 options: Options::new(),
683 not_failures: Vec::new(),
684 time_failures: Vec::new(),
687 out.write_failures(&st).unwrap();
688 let s = match out.output_location() {
689 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
690 &OutputLocation::Pretty(_) => unreachable!(),
693 let apos = s.find("a").unwrap();
694 let bpos = s.find("b").unwrap();
695 assert!(apos < bpos);