5 console::OutputLocation,
7 time::{TimeThreshold, TestTimeOptions},
8 formatters::PrettyFormatter,
10 filter_tests, parse_opts, run_test, DynTestFn, DynTestName, MetricMap,
11 RunIgnored, RunStrategy, ShouldPanic, StaticTestName, TestDesc,
12 TestDescAndFn, TestOpts, TrIgnored, TrOk,
13 // FIXME (introduced by #65251)
14 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
15 // TestType, TrFailedMsg, TrIgnored, TrOk,
18 use std::sync::mpsc::channel;
19 use std::time::Duration;
22 fn new() -> TestOpts {
27 force_run_in_process: false,
28 exclude_should_panic: false,
29 run_ignored: RunIgnored::No,
31 bench_benchmarks: false,
35 format: OutputFormat::Pretty,
39 options: Options::new(),
44 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
48 name: StaticTestName("1"),
50 should_panic: ShouldPanic::No,
52 test_type: TestType::Unknown,
54 testfn: DynTestFn(Box::new(move || {})),
58 name: StaticTestName("2"),
60 should_panic: ShouldPanic::No,
62 test_type: TestType::Unknown,
64 testfn: DynTestFn(Box::new(move || {})),
70 pub fn do_not_run_ignored_tests() {
74 let desc = TestDescAndFn {
76 name: StaticTestName("whatever"),
78 should_panic: ShouldPanic::No,
80 test_type: TestType::Unknown,
82 testfn: DynTestFn(Box::new(f)),
84 let (tx, rx) = channel();
85 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
86 let result = rx.recv().unwrap().result;
87 assert!(result != TrOk);
91 pub fn ignored_tests_result_in_ignored() {
93 let desc = TestDescAndFn {
95 name: StaticTestName("whatever"),
97 should_panic: ShouldPanic::No,
99 test_type: TestType::Unknown,
101 testfn: DynTestFn(Box::new(f)),
103 let (tx, rx) = channel();
104 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
105 let result = rx.recv().unwrap().result;
106 assert!(result == TrIgnored);
109 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
111 #[cfg(not(target_os = "emscripten"))]
112 fn test_should_panic() {
116 let desc = TestDescAndFn {
118 name: StaticTestName("whatever"),
120 should_panic: ShouldPanic::Yes,
122 test_type: TestType::Unknown,
124 testfn: DynTestFn(Box::new(f)),
126 let (tx, rx) = channel();
127 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
128 let result = rx.recv().unwrap().result;
129 assert!(result == TrOk);
132 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
134 #[cfg(not(target_os = "emscripten"))]
135 fn test_should_panic_good_message() {
137 panic!("an error message");
139 let desc = TestDescAndFn {
141 name: StaticTestName("whatever"),
143 should_panic: ShouldPanic::YesWithMessage("error message"),
145 test_type: TestType::Unknown,
147 testfn: DynTestFn(Box::new(f)),
149 let (tx, rx) = channel();
150 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
151 let result = rx.recv().unwrap().result;
152 assert!(result == TrOk);
155 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
157 #[cfg(not(target_os = "emscripten"))]
158 fn test_should_panic_bad_message() {
159 use crate::tests::TrFailedMsg;
161 panic!("an error message");
163 let expected = "foobar";
164 let failed_msg = "panic did not include expected string";
165 let desc = TestDescAndFn {
167 name: StaticTestName("whatever"),
169 should_panic: ShouldPanic::YesWithMessage(expected),
171 test_type: TestType::Unknown,
173 testfn: DynTestFn(Box::new(f)),
175 let (tx, rx) = channel();
176 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
177 let result = rx.recv().unwrap().result;
178 assert!(result == TrFailedMsg(format!("{} '{}'", failed_msg, expected)));
181 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
183 #[cfg(not(target_os = "emscripten"))]
184 fn test_should_panic_but_succeeds() {
186 let desc = TestDescAndFn {
188 name: StaticTestName("whatever"),
190 should_panic: ShouldPanic::Yes,
192 test_type: TestType::Unknown,
194 testfn: DynTestFn(Box::new(f)),
196 let (tx, rx) = channel();
197 run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
198 let result = rx.recv().unwrap().result;
199 assert!(result == TrFailedMsg("test did not panic as expected".to_string()));
202 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
204 let desc = TestDescAndFn {
206 name: StaticTestName("whatever"),
208 should_panic: ShouldPanic::No,
210 test_type: TestType::Unknown,
212 testfn: DynTestFn(Box::new(f)),
214 let time_options = if report_time {
215 Some(TestTimeOptions::default())
220 let test_opts = TestOpts {
224 let (tx, rx) = channel();
225 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
226 let exec_time = rx.recv().unwrap().exec_time;
231 fn test_should_not_report_time() {
232 let exec_time = report_time_test_template(false);
233 assert!(exec_time.is_none());
237 fn test_should_report_time() {
238 let exec_time = report_time_test_template(true);
239 assert!(exec_time.is_some());
242 fn time_test_failure_template(test_type: TestType) -> TestResult {
244 let desc = TestDescAndFn {
246 name: StaticTestName("whatever"),
248 should_panic: ShouldPanic::No,
252 testfn: DynTestFn(Box::new(f)),
254 // `Default` will initialize all the thresholds to 0 milliseconds.
255 let mut time_options = TestTimeOptions::default();
256 time_options.error_on_excess = true;
258 let test_opts = TestOpts {
259 time_options: Some(time_options),
262 let (tx, rx) = channel();
263 run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
264 let result = rx.recv().unwrap().result;
270 fn test_error_on_exceed() {
271 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
273 for test_type in types.iter() {
274 let result = time_test_failure_template(*test_type);
276 assert_eq!(result, TestResult::TrTimedFail);
279 // Check that for unknown tests thresholds aren't applied.
280 let result = time_test_failure_template(TestType::Unknown);
281 assert_eq!(result, TestResult::TrOk);
284 fn typed_test_desc(test_type: TestType) -> TestDesc {
286 name: StaticTestName("whatever"),
288 should_panic: ShouldPanic::No,
294 fn test_exec_time(millis: u64) -> TestExecTime {
295 TestExecTime(Duration::from_millis(millis))
299 fn test_time_options_threshold() {
300 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
301 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
302 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
304 let options = TestTimeOptions {
305 error_on_excess: false,
307 unit_threshold: unit.clone(),
308 integration_threshold: integration.clone(),
309 doctest_threshold: doc.clone(),
313 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
314 (TestType::UnitTest, unit.warn.as_millis(), true, false),
315 (TestType::UnitTest, unit.critical.as_millis(), true, true),
316 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
317 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
318 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
319 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
320 (TestType::DocTest, doc.warn.as_millis(), true, false),
321 (TestType::DocTest, doc.critical.as_millis(), true, true),
324 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
325 let test_desc = typed_test_desc(*test_type);
326 let exec_time = test_exec_time(*time as u64);
328 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
329 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
334 fn parse_ignored_flag() {
336 "progname".to_string(),
337 "filter".to_string(),
338 "--ignored".to_string(),
340 let opts = parse_opts(&args).unwrap().unwrap();
341 assert_eq!(opts.run_ignored, RunIgnored::Only);
345 fn parse_show_output_flag() {
347 "progname".to_string(),
348 "filter".to_string(),
349 "--show-output".to_string(),
351 let opts = parse_opts(&args).unwrap().unwrap();
352 assert!(opts.options.display_output);
356 fn parse_include_ignored_flag() {
358 "progname".to_string(),
359 "filter".to_string(),
360 "-Zunstable-options".to_string(),
361 "--include-ignored".to_string(),
363 let opts = parse_opts(&args).unwrap().unwrap();
364 assert_eq!(opts.run_ignored, RunIgnored::Yes);
368 pub fn filter_for_ignored_option() {
369 // When we run ignored tests the test filter should filter out all the
370 // unignored tests and flip the ignore flag on the rest to false
372 let mut opts = TestOpts::new();
373 opts.run_tests = true;
374 opts.run_ignored = RunIgnored::Only;
376 let tests = one_ignored_one_unignored_test();
377 let filtered = filter_tests(&opts, tests);
379 assert_eq!(filtered.len(), 1);
380 assert_eq!(filtered[0].desc.name.to_string(), "1");
381 assert!(!filtered[0].desc.ignore);
385 pub fn run_include_ignored_option() {
386 // When we "--include-ignored" tests, the ignore flag should be set to false on
387 // all tests and no test filtered out
389 let mut opts = TestOpts::new();
390 opts.run_tests = true;
391 opts.run_ignored = RunIgnored::Yes;
393 let tests = one_ignored_one_unignored_test();
394 let filtered = filter_tests(&opts, tests);
396 assert_eq!(filtered.len(), 2);
397 assert!(!filtered[0].desc.ignore);
398 assert!(!filtered[1].desc.ignore);
402 pub fn exclude_should_panic_option() {
403 let mut opts = TestOpts::new();
404 opts.run_tests = true;
405 opts.exclude_should_panic = true;
407 let mut tests = one_ignored_one_unignored_test();
408 tests.push(TestDescAndFn {
410 name: StaticTestName("3"),
412 should_panic: ShouldPanic::Yes,
414 test_type: TestType::Unknown,
416 testfn: DynTestFn(Box::new(move || {})),
419 let filtered = filter_tests(&opts, tests);
421 assert_eq!(filtered.len(), 2);
422 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
426 pub fn exact_filter_match() {
427 fn tests() -> Vec<TestDescAndFn> {
428 vec!["base", "base::test", "base::test1", "base::test2"]
430 .map(|name| TestDescAndFn {
432 name: StaticTestName(name),
434 should_panic: ShouldPanic::No,
436 test_type: TestType::Unknown,
438 testfn: DynTestFn(Box::new(move || {})),
443 let substr = filter_tests(
445 filter: Some("base".into()),
450 assert_eq!(substr.len(), 4);
452 let substr = filter_tests(
454 filter: Some("bas".into()),
459 assert_eq!(substr.len(), 4);
461 let substr = filter_tests(
463 filter: Some("::test".into()),
468 assert_eq!(substr.len(), 3);
470 let substr = filter_tests(
472 filter: Some("base::test".into()),
477 assert_eq!(substr.len(), 3);
479 let exact = filter_tests(
481 filter: Some("base".into()),
487 assert_eq!(exact.len(), 1);
489 let exact = filter_tests(
491 filter: Some("bas".into()),
497 assert_eq!(exact.len(), 0);
499 let exact = filter_tests(
501 filter: Some("::test".into()),
507 assert_eq!(exact.len(), 0);
509 let exact = filter_tests(
511 filter: Some("base::test".into()),
517 assert_eq!(exact.len(), 1);
521 pub fn sort_tests() {
522 let mut opts = TestOpts::new();
523 opts.run_tests = true;
526 "sha1::test".to_string(),
527 "isize::test_to_str".to_string(),
528 "isize::test_pow".to_string(),
529 "test::do_not_run_ignored_tests".to_string(),
530 "test::ignored_tests_result_in_ignored".to_string(),
531 "test::first_free_arg_should_be_a_filter".to_string(),
532 "test::parse_ignored_flag".to_string(),
533 "test::parse_include_ignored_flag".to_string(),
534 "test::filter_for_ignored_option".to_string(),
535 "test::run_include_ignored_option".to_string(),
536 "test::sort_tests".to_string(),
540 let mut tests = Vec::new();
542 let test = TestDescAndFn {
544 name: DynTestName((*name).clone()),
546 should_panic: ShouldPanic::No,
548 test_type: TestType::Unknown,
550 testfn: DynTestFn(Box::new(testfn)),
556 let filtered = filter_tests(&opts, tests);
559 "isize::test_pow".to_string(),
560 "isize::test_to_str".to_string(),
561 "sha1::test".to_string(),
562 "test::do_not_run_ignored_tests".to_string(),
563 "test::filter_for_ignored_option".to_string(),
564 "test::first_free_arg_should_be_a_filter".to_string(),
565 "test::ignored_tests_result_in_ignored".to_string(),
566 "test::parse_ignored_flag".to_string(),
567 "test::parse_include_ignored_flag".to_string(),
568 "test::run_include_ignored_option".to_string(),
569 "test::sort_tests".to_string(),
572 for (a, b) in expected.iter().zip(filtered) {
573 assert!(*a == b.desc.name.to_string());
578 pub fn test_metricmap_compare() {
579 let mut m1 = MetricMap::new();
580 let mut m2 = MetricMap::new();
581 m1.insert_metric("in-both-noise", 1000.0, 200.0);
582 m2.insert_metric("in-both-noise", 1100.0, 200.0);
584 m1.insert_metric("in-first-noise", 1000.0, 2.0);
585 m2.insert_metric("in-second-noise", 1000.0, 2.0);
587 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
588 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
590 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
591 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
593 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
594 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
596 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
597 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
601 pub fn test_bench_once_no_iter() {
602 fn f(_: &mut Bencher) {}
607 pub fn test_bench_once_iter() {
608 fn f(b: &mut Bencher) {
615 pub fn test_bench_no_iter() {
616 fn f(_: &mut Bencher) {}
618 let (tx, rx) = channel();
620 let desc = TestDesc {
621 name: StaticTestName("f"),
623 should_panic: ShouldPanic::No,
625 test_type: TestType::Unknown,
628 crate::bench::benchmark(desc, tx, true, f);
633 pub fn test_bench_iter() {
634 fn f(b: &mut Bencher) {
638 let (tx, rx) = channel();
640 let desc = TestDesc {
641 name: StaticTestName("f"),
643 should_panic: ShouldPanic::No,
645 test_type: TestType::Unknown,
648 crate::bench::benchmark(desc, tx, true, f);
653 fn should_sort_failures_before_printing_them() {
654 let test_a = TestDesc {
655 name: StaticTestName("a"),
657 should_panic: ShouldPanic::No,
659 test_type: TestType::Unknown,
662 let test_b = TestDesc {
663 name: StaticTestName("b"),
665 should_panic: ShouldPanic::No,
667 test_type: TestType::Unknown,
670 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
672 let st = console::ConsoleTestState {
681 metrics: MetricMap::new(),
682 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
683 options: Options::new(),
684 not_failures: Vec::new(),
685 time_failures: Vec::new(),
688 out.write_failures(&st).unwrap();
689 let s = match out.output_location() {
690 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
691 &OutputLocation::Pretty(_) => unreachable!(),
694 let apos = s.find("a").unwrap();
695 let bpos = s.find("b").unwrap();
696 assert!(apos < bpos);