1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Support code for rustc's built in unit-test and micro-benchmarking
14 //! Almost all user code will only be interested in `Bencher` and
15 //! `black_box`. All other interactions (such as writing tests and
16 //! benchmarks themselves) should be done via the `#[test]` and
17 //! `#[bench]` attributes.
19 //! See the [Testing Chapter](../book/testing.html) of the book for more details.
21 // Currently, not much of this is meant for users. It is intended to
22 // support the simplest interface possible for representing and
23 // running tests while providing a base that other test frameworks may
26 #![crate_name = "test"]
27 #![unstable(feature = "test", issue = "27812")]
28 #![crate_type = "rlib"]
29 #![crate_type = "dylib"]
30 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
31 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
32 html_root_url = "https://doc.rust-lang.org/nightly/",
33 test(attr(deny(warnings))))]
34 #![cfg_attr(not(stage0), deny(warnings))]
37 #![feature(box_syntax)]
40 #![feature(rustc_private)]
41 #![feature(set_stdio)]
42 #![feature(staged_api)]
46 extern crate serialize;
47 extern crate serialize as rustc_serialize;
51 pub use self::TestFn::*;
52 pub use self::ColorConfig::*;
53 pub use self::TestResult::*;
54 pub use self::TestName::*;
55 use self::TestEvent::*;
56 use self::NamePadding::*;
57 use self::OutputLocation::*;
60 use serialize::Encodable;
61 use std::boxed::FnBox;
66 use std::collections::BTreeMap;
70 use std::io::prelude::*;
72 use std::iter::repeat;
73 use std::path::PathBuf;
74 use std::sync::mpsc::{channel, Sender};
75 use std::sync::{Arc, Mutex};
77 use std::time::{Instant, Duration};
79 // to be used by rustc to compile tests in libtest
81 pub use {Bencher, TestName, TestResult, TestDesc, TestDescAndFn, TestOpts, TrFailed,
82 TrIgnored, TrOk, Metric, MetricMap, StaticTestFn, StaticTestName, DynTestName,
83 DynTestFn, run_test, test_main, test_main_static, filter_tests, parse_opts,
84 StaticBenchFn, ShouldPanic};
89 // The name of a test. By convention this follows the rules for rust
90 // paths; i.e. it should be a series of identifiers separated by double
91 // colons. This way if some test runner wants to arrange the tests
92 // hierarchically it may.
94 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
96 StaticTestName(&'static str),
100 fn as_slice(&self) -> &str {
102 StaticTestName(s) => s,
103 DynTestName(ref s) => s,
107 impl fmt::Display for TestName {
108 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
109 fmt::Display::fmt(self.as_slice(), f)
113 #[derive(Clone, Copy)]
120 fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
121 let mut name = String::from(self.name.as_slice());
122 let fill = column_count.saturating_sub(name.len());
123 let pad = repeat(" ").take(fill).collect::<String>();
134 /// Represents a benchmark function.
135 pub trait TDynBenchFn: Send {
136 fn run(&self, harness: &mut Bencher);
139 // A function that runs a test. If the function returns successfully,
140 // the test succeeds; if the function panics then the test fails. We
141 // may need to come up with a more clever definition of test in order
142 // to support isolation of tests into threads.
145 StaticBenchFn(fn(&mut Bencher)),
146 StaticMetricFn(fn(&mut MetricMap)),
147 DynTestFn(Box<FnBox() + Send>),
148 DynMetricFn(Box<FnBox(&mut MetricMap) + Send>),
149 DynBenchFn(Box<TDynBenchFn + 'static>),
153 fn padding(&self) -> NamePadding {
155 StaticTestFn(..) => PadNone,
156 StaticBenchFn(..) => PadOnRight,
157 StaticMetricFn(..) => PadOnRight,
158 DynTestFn(..) => PadNone,
159 DynMetricFn(..) => PadOnRight,
160 DynBenchFn(..) => PadOnRight,
165 impl fmt::Debug for TestFn {
166 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
167 f.write_str(match *self {
168 StaticTestFn(..) => "StaticTestFn(..)",
169 StaticBenchFn(..) => "StaticBenchFn(..)",
170 StaticMetricFn(..) => "StaticMetricFn(..)",
171 DynTestFn(..) => "DynTestFn(..)",
172 DynMetricFn(..) => "DynMetricFn(..)",
173 DynBenchFn(..) => "DynBenchFn(..)",
178 /// Manager of the benchmarking runs.
180 /// This is fed into functions marked with `#[bench]` to allow for
181 /// set-up & tear-down before running a piece of code repeatedly via a
183 #[derive(Copy, Clone)]
190 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
191 pub enum ShouldPanic {
194 YesWithMessage(&'static str),
197 // The definition of a single test. A test runner will run a list of
199 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
200 pub struct TestDesc {
203 pub should_panic: ShouldPanic,
206 unsafe impl Send for TestDesc {}
209 pub struct TestDescAndFn {
214 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug, Copy)]
221 pub fn new(value: f64, noise: f64) -> Metric {
230 pub struct MetricMap(BTreeMap<String, Metric>);
232 impl Clone for MetricMap {
233 fn clone(&self) -> MetricMap {
234 let MetricMap(ref map) = *self;
235 MetricMap(map.clone())
239 // The default console test runner. It accepts the command line
240 // arguments and a vector of test_descs.
241 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>) {
242 let opts = match parse_opts(args) {
244 Some(Err(msg)) => panic!("{:?}", msg),
247 match run_tests_console(&opts, tests) {
249 Ok(false) => std::process::exit(101),
250 Err(e) => panic!("io error when running tests: {:?}", e),
254 // A variant optimized for invocation with a static test vector.
255 // This will panic (intentionally) when fed any dynamic tests, because
256 // it is copying the static values out into a dynamic vector and cannot
257 // copy dynamic values. It is doing this because from this point on
258 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
259 // semantics into parallel test runners, which in turn requires a Vec<>
260 // rather than a &[].
261 pub fn test_main_static(tests: &[TestDescAndFn]) {
262 let args = env::args().collect::<Vec<_>>();
263 let owned_tests = tests.iter()
268 testfn: StaticTestFn(f),
269 desc: t.desc.clone(),
272 StaticBenchFn(f) => {
274 testfn: StaticBenchFn(f),
275 desc: t.desc.clone(),
278 _ => panic!("non-static tests passed to test::test_main_static"),
282 test_main(&args, owned_tests)
285 #[derive(Copy, Clone)]
286 pub enum ColorConfig {
292 pub struct TestOpts {
293 pub filter: Option<String>,
294 pub run_ignored: bool,
296 pub bench_benchmarks: bool,
297 pub logfile: Option<PathBuf>,
299 pub color: ColorConfig,
304 fn new() -> TestOpts {
309 bench_benchmarks: false,
317 /// Result of parsing the options.
318 pub type OptRes = Result<TestOpts, String>;
320 #[cfg_attr(rustfmt, rustfmt_skip)]
321 fn optgroups() -> Vec<getopts::OptGroup> {
322 vec!(getopts::optflag("", "ignored", "Run ignored tests"),
323 getopts::optflag("", "test", "Run tests and not benchmarks"),
324 getopts::optflag("", "bench", "Run benchmarks instead of tests"),
325 getopts::optflag("h", "help", "Display this message (longer with --help)"),
326 getopts::optopt("", "logfile", "Write logs to the specified file instead \
328 getopts::optflag("", "nocapture", "don't capture stdout/stderr of each \
329 task, allow printing directly"),
330 getopts::optopt("", "color", "Configure coloring of output:
331 auto = colorize if stdout is a tty and tests are run on serially (default);
332 always = always colorize output;
333 never = never colorize output;", "auto|always|never"))
336 fn usage(binary: &str) {
337 let message = format!("Usage: {} [OPTIONS] [FILTER]", binary);
340 The FILTER string is tested against the name of all tests, and only those
341 tests whose names contain the filter are run.
343 By default, all tests are run in parallel. This can be altered with the
344 RUST_TEST_THREADS environment variable when running tests (set it to 1).
346 All tests have their standard output and standard error captured by default.
347 This can be overridden with the --nocapture flag or the RUST_TEST_NOCAPTURE=1
348 environment variable. Logging is not captured by default.
352 #[test] - Indicates a function is a test to be run. This function
354 #[bench] - Indicates a function is a benchmark to be run. This
355 function takes one argument (test::Bencher).
356 #[should_panic] - This function (also labeled with #[test]) will only pass if
357 the code causes a panic (an assertion failure or panic!)
358 A message may be provided, which the failure string must
359 contain: #[should_panic(expected = "foo")].
360 #[ignore] - When applied to a function which is already attributed as a
361 test, then the test runner will ignore these tests during
362 normal test runs. Running with --ignored will run these
364 usage = getopts::usage(&message, &optgroups()));
367 // Parses command line arguments into test options
368 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
369 let args_ = &args[1..];
370 let matches = match getopts::getopts(args_, &optgroups()) {
372 Err(f) => return Some(Err(f.to_string())),
375 if matches.opt_present("h") {
380 let filter = if !matches.free.is_empty() {
381 Some(matches.free[0].clone())
386 let run_ignored = matches.opt_present("ignored");
388 let logfile = matches.opt_str("logfile");
389 let logfile = logfile.map(|s| PathBuf::from(&s));
391 let bench_benchmarks = matches.opt_present("bench");
392 let run_tests = !bench_benchmarks || matches.opt_present("test");
394 let mut nocapture = matches.opt_present("nocapture");
396 nocapture = env::var("RUST_TEST_NOCAPTURE").is_ok();
399 let color = match matches.opt_str("color").as_ref().map(|s| &**s) {
400 Some("auto") | None => AutoColor,
401 Some("always") => AlwaysColor,
402 Some("never") => NeverColor,
405 return Some(Err(format!("argument for --color must be auto, always, or never (was \
411 let test_opts = TestOpts {
413 run_ignored: run_ignored,
414 run_tests: run_tests,
415 bench_benchmarks: bench_benchmarks,
417 nocapture: nocapture,
424 #[derive(Clone, PartialEq)]
425 pub struct BenchSamples {
426 ns_iter_summ: stats::Summary,
430 #[derive(Clone, PartialEq)]
431 pub enum TestResult {
435 TrMetrics(MetricMap),
436 TrBench(BenchSamples),
439 unsafe impl Send for TestResult {}
441 enum OutputLocation<T> {
442 Pretty(Box<term::StdoutTerminal>),
446 struct ConsoleTestState<T> {
447 log_out: Option<File>,
448 out: OutputLocation<T>,
456 failures: Vec<(TestDesc, Vec<u8>)>,
457 max_name_len: usize, // number of columns to fill when aligning names
460 impl<T: Write> ConsoleTestState<T> {
461 pub fn new(opts: &TestOpts, _: Option<T>) -> io::Result<ConsoleTestState<io::Stdout>> {
462 let log_out = match opts.logfile {
463 Some(ref path) => Some(try!(File::create(path))),
466 let out = match term::stdout() {
467 None => Raw(io::stdout()),
468 Some(t) => Pretty(t),
471 Ok(ConsoleTestState {
474 use_color: use_color(opts),
480 metrics: MetricMap::new(),
481 failures: Vec::new(),
486 pub fn write_ok(&mut self) -> io::Result<()> {
487 self.write_pretty("ok", term::color::GREEN)
490 pub fn write_failed(&mut self) -> io::Result<()> {
491 self.write_pretty("FAILED", term::color::RED)
494 pub fn write_ignored(&mut self) -> io::Result<()> {
495 self.write_pretty("ignored", term::color::YELLOW)
498 pub fn write_metric(&mut self) -> io::Result<()> {
499 self.write_pretty("metric", term::color::CYAN)
502 pub fn write_bench(&mut self) -> io::Result<()> {
503 self.write_pretty("bench", term::color::CYAN)
506 pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
508 Pretty(ref mut term) => {
510 try!(term.fg(color));
512 try!(term.write_all(word.as_bytes()));
518 Raw(ref mut stdout) => {
519 try!(stdout.write_all(word.as_bytes()));
525 pub fn write_plain(&mut self, s: &str) -> io::Result<()> {
527 Pretty(ref mut term) => {
528 try!(term.write_all(s.as_bytes()));
531 Raw(ref mut stdout) => {
532 try!(stdout.write_all(s.as_bytes()));
538 pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
540 let noun = if len != 1 {
545 self.write_plain(&format!("\nrunning {} {}\n", len, noun))
548 pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
549 let name = test.padded_name(self.max_name_len, align);
550 self.write_plain(&format!("test {} ... ", name))
553 pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
555 TrOk => self.write_ok(),
556 TrFailed => self.write_failed(),
557 TrIgnored => self.write_ignored(),
558 TrMetrics(ref mm) => {
559 try!(self.write_metric());
560 self.write_plain(&format!(": {}", mm.fmt_metrics()))
563 try!(self.write_bench());
565 try!(self.write_plain(&format!(": {}", fmt_bench_samples(bs))));
570 self.write_plain("\n")
573 pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
577 let s = format!("{} {}\n",
579 TrOk => "ok".to_owned(),
580 TrFailed => "failed".to_owned(),
581 TrIgnored => "ignored".to_owned(),
582 TrMetrics(ref mm) => mm.fmt_metrics(),
583 TrBench(ref bs) => fmt_bench_samples(bs),
586 o.write_all(s.as_bytes())
591 pub fn write_failures(&mut self) -> io::Result<()> {
592 try!(self.write_plain("\nfailures:\n"));
593 let mut failures = Vec::new();
594 let mut fail_out = String::new();
595 for &(ref f, ref stdout) in &self.failures {
596 failures.push(f.name.to_string());
597 if !stdout.is_empty() {
598 fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
599 let output = String::from_utf8_lossy(stdout);
600 fail_out.push_str(&output);
601 fail_out.push_str("\n");
604 if !fail_out.is_empty() {
605 try!(self.write_plain("\n"));
606 try!(self.write_plain(&fail_out));
609 try!(self.write_plain("\nfailures:\n"));
611 for name in &failures {
612 try!(self.write_plain(&format!(" {}\n", name)));
617 pub fn write_run_finish(&mut self) -> io::Result<bool> {
618 assert!(self.passed + self.failed + self.ignored + self.measured == self.total);
620 let success = self.failed == 0;
622 try!(self.write_failures());
625 try!(self.write_plain("\ntest result: "));
627 // There's no parallelism at this point so it's safe to use color
628 try!(self.write_ok());
630 try!(self.write_failed());
632 let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n",
637 try!(self.write_plain(&s));
642 // Format a number with thousands separators
643 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
645 let mut output = String::new();
646 let mut trailing = false;
647 for &pow in &[9, 6, 3, 0] {
648 let base = 10_usize.pow(pow);
649 if pow == 0 || trailing || n / base != 0 {
651 output.write_fmt(format_args!("{}", n / base)).unwrap();
653 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
666 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
668 let mut output = String::new();
670 let median = bs.ns_iter_summ.median as usize;
671 let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
673 output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
674 fmt_thousands_sep(median, ','),
675 fmt_thousands_sep(deviation, ',')))
678 output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
683 // A simple console test runner
684 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
686 fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
687 match (*event).clone() {
688 TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
689 TeWait(ref test, padding) => st.write_test_start(test, padding),
690 TeResult(test, result, stdout) => {
691 try!(st.write_log(&test, &result));
692 try!(st.write_result(&result));
694 TrOk => st.passed += 1,
695 TrIgnored => st.ignored += 1,
697 let tname = test.name;
698 let MetricMap(mm) = mm;
701 .insert_metric(&format!("{}.{}", tname, k), v.value, v.noise);
706 st.metrics.insert_metric(test.name.as_slice(),
707 bs.ns_iter_summ.median,
708 bs.ns_iter_summ.max - bs.ns_iter_summ.min);
713 st.failures.push((test, stdout));
721 let mut st = try!(ConsoleTestState::new(opts, None::<io::Stdout>));
722 fn len_if_padded(t: &TestDescAndFn) -> usize {
723 match t.testfn.padding() {
725 PadOnRight => t.desc.name.as_slice().len(),
728 match tests.iter().max_by_key(|t| len_if_padded(*t)) {
730 let n = t.desc.name.as_slice();
731 st.max_name_len = n.len();
735 try!(run_tests(opts, tests, |x| callback(&x, &mut st)));
736 return st.write_run_finish();
740 fn should_sort_failures_before_printing_them() {
741 let test_a = TestDesc {
742 name: StaticTestName("a"),
744 should_panic: ShouldPanic::No,
747 let test_b = TestDesc {
748 name: StaticTestName("b"),
750 should_panic: ShouldPanic::No,
753 let mut st = ConsoleTestState {
755 out: Raw(Vec::new()),
763 metrics: MetricMap::new(),
764 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
767 st.write_failures().unwrap();
768 let s = match st.out {
769 Raw(ref m) => String::from_utf8_lossy(&m[..]),
770 Pretty(_) => unreachable!(),
773 let apos = s.find("a").unwrap();
774 let bpos = s.find("b").unwrap();
775 assert!(apos < bpos);
778 fn use_color(opts: &TestOpts) -> bool {
780 AutoColor => !opts.nocapture && stdout_isatty(),
787 fn stdout_isatty() -> bool {
788 unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
791 fn stdout_isatty() -> bool {
794 type HANDLE = *mut u8;
795 type LPDWORD = *mut u32;
796 const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD;
798 fn GetStdHandle(which: DWORD) -> HANDLE;
799 fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL;
802 let handle = GetStdHandle(STD_OUTPUT_HANDLE);
804 GetConsoleMode(handle, &mut out) != 0
810 TeFiltered(Vec<TestDesc>),
811 TeWait(TestDesc, NamePadding),
812 TeResult(TestDesc, TestResult, Vec<u8>),
815 pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
818 fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
819 where F: FnMut(TestEvent) -> io::Result<()>
821 let mut filtered_tests = filter_tests(opts, tests);
822 if !opts.bench_benchmarks {
823 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
826 let filtered_descs = filtered_tests.iter()
827 .map(|t| t.desc.clone())
830 try!(callback(TeFiltered(filtered_descs)));
832 let (filtered_tests, filtered_benchs_and_metrics): (Vec<_>, _) =
833 filtered_tests.into_iter().partition(|e| {
835 StaticTestFn(_) | DynTestFn(_) => true,
840 // It's tempting to just spawn all the tests at once, but since we have
841 // many tests that run in other processes we would be making a big mess.
842 let concurrency = get_concurrency();
844 let mut remaining = filtered_tests;
848 let (tx, rx) = channel::<MonitorMsg>();
850 while pending > 0 || !remaining.is_empty() {
851 while pending < concurrency && !remaining.is_empty() {
852 let test = remaining.pop().unwrap();
853 if concurrency == 1 {
854 // We are doing one test at a time so we can print the name
855 // of the test before we run it. Useful for debugging tests
856 // that hang forever.
857 try!(callback(TeWait(test.desc.clone(), test.testfn.padding())));
859 run_test(opts, !opts.run_tests, test, tx.clone());
863 let (desc, result, stdout) = rx.recv().unwrap();
864 if concurrency != 1 {
865 try!(callback(TeWait(desc.clone(), PadNone)));
867 try!(callback(TeResult(desc, result, stdout)));
871 if opts.bench_benchmarks {
872 // All benchmarks run at the end, in serial.
873 // (this includes metric fns)
874 for b in filtered_benchs_and_metrics {
875 try!(callback(TeWait(b.desc.clone(), b.testfn.padding())));
876 run_test(opts, false, b, tx.clone());
877 let (test, result, stdout) = rx.recv().unwrap();
878 try!(callback(TeResult(test, result, stdout)));
885 fn get_concurrency() -> usize {
886 return match env::var("RUST_TEST_THREADS") {
888 let opt_n: Option<usize> = s.parse().ok();
890 Some(n) if n > 0 => n,
892 panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.",
897 Err(..) => num_cpus(),
902 fn num_cpus() -> usize {
905 wProcessorArchitecture: u16,
908 lpMinimumApplicationAddress: *mut u8,
909 lpMaximumApplicationAddress: *mut u8,
910 dwActiveProcessorMask: *mut u8,
911 dwNumberOfProcessors: u32,
912 dwProcessorType: u32,
913 dwAllocationGranularity: u32,
914 wProcessorLevel: u16,
915 wProcessorRevision: u16,
918 fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32;
921 let mut sysinfo = std::mem::zeroed();
922 GetSystemInfo(&mut sysinfo);
923 sysinfo.dwNumberOfProcessors as usize
927 #[cfg(any(target_os = "linux",
930 target_os = "android"))]
931 fn num_cpus() -> usize {
932 unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
935 #[cfg(any(target_os = "freebsd",
936 target_os = "dragonfly",
937 target_os = "bitrig",
938 target_os = "netbsd"))]
939 fn num_cpus() -> usize {
940 let mut cpus: libc::c_uint = 0;
941 let mut cpus_size = std::mem::size_of_val(&cpus);
944 cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint;
947 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
949 libc::sysctl(mib.as_mut_ptr(),
951 &mut cpus as *mut _ as *mut _,
952 &mut cpus_size as *mut _ as *mut _,
963 #[cfg(target_os = "openbsd")]
964 fn num_cpus() -> usize {
965 let mut cpus: libc::c_uint = 0;
966 let mut cpus_size = std::mem::size_of_val(&cpus);
967 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
970 libc::sysctl(mib.as_mut_ptr(),
972 &mut cpus as *mut _ as *mut _,
973 &mut cpus_size as *mut _ as *mut _,
984 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
985 let mut filtered = tests;
987 // Remove tests that don't match the test filter
988 filtered = match opts.filter {
990 Some(ref filter) => {
992 .filter(|test| test.desc.name.as_slice().contains(&filter[..]))
997 // Maybe pull out the ignored test and unignore them
998 filtered = if !opts.run_ignored {
1001 fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
1002 if test.desc.ignore {
1003 let TestDescAndFn {desc, testfn} = test;
1004 Some(TestDescAndFn {
1005 desc: TestDesc { ignore: false, ..desc },
1012 filtered.into_iter().filter_map(filter).collect()
1015 // Sort the tests alphabetically
1016 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
1021 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1022 // convert benchmarks to tests, if we're not benchmarking them
1025 let testfn = match x.testfn {
1026 DynBenchFn(bench) => {
1027 DynTestFn(Box::new(move || bench::run_once(|b| bench.run(b))))
1029 StaticBenchFn(benchfn) => {
1030 DynTestFn(Box::new(move || bench::run_once(|b| benchfn(b))))
1042 pub fn run_test(opts: &TestOpts,
1044 test: TestDescAndFn,
1045 monitor_ch: Sender<MonitorMsg>) {
1047 let TestDescAndFn {desc, testfn} = test;
1049 if force_ignore || desc.ignore {
1050 monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap();
1054 fn run_test_inner(desc: TestDesc,
1055 monitor_ch: Sender<MonitorMsg>,
1057 testfn: Box<FnBox() + Send>) {
1058 struct Sink(Arc<Mutex<Vec<u8>>>);
1059 impl Write for Sink {
1060 fn write(&mut self, data: &[u8]) -> io::Result<usize> {
1061 Write::write(&mut *self.0.lock().unwrap(), data)
1063 fn flush(&mut self) -> io::Result<()> {
1068 thread::spawn(move || {
1069 let data = Arc::new(Mutex::new(Vec::new()));
1070 let data2 = data.clone();
1071 let cfg = thread::Builder::new().name(match desc.name {
1072 DynTestName(ref name) => name.clone(),
1073 StaticTestName(name) => name.to_owned(),
1076 let result_guard = cfg.spawn(move || {
1078 io::set_print(box Sink(data2.clone()));
1079 io::set_panic(box Sink(data2));
1084 let test_result = calc_result(&desc, result_guard.join());
1085 let stdout = data.lock().unwrap().to_vec();
1086 monitor_ch.send((desc.clone(), test_result, stdout)).unwrap();
1091 DynBenchFn(bencher) => {
1092 let bs = ::bench::benchmark(|harness| bencher.run(harness));
1093 monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
1096 StaticBenchFn(benchfn) => {
1097 let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness));
1098 monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
1102 let mut mm = MetricMap::new();
1103 f.call_box((&mut mm,));
1104 monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap();
1107 StaticMetricFn(f) => {
1108 let mut mm = MetricMap::new();
1110 monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap();
1113 DynTestFn(f) => run_test_inner(desc, monitor_ch, opts.nocapture, f),
1114 StaticTestFn(f) => run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(f)),
1118 fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> TestResult {
1119 match (&desc.should_panic, task_result) {
1120 (&ShouldPanic::No, Ok(())) |
1121 (&ShouldPanic::Yes, Err(_)) => TrOk,
1122 (&ShouldPanic::YesWithMessage(msg), Err(ref err))
1123 if err.downcast_ref::<String>()
1125 .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
1126 .map(|e| e.contains(msg))
1127 .unwrap_or(false) => TrOk,
1133 pub fn new() -> MetricMap {
1134 MetricMap(BTreeMap::new())
1137 /// Insert a named `value` (+/- `noise`) metric into the map. The value
1138 /// must be non-negative. The `noise` indicates the uncertainty of the
1139 /// metric, which doubles as the "noise range" of acceptable
1140 /// pairwise-regressions on this named value, when comparing from one
1141 /// metric to the next using `compare_to_old`.
1143 /// If `noise` is positive, then it means this metric is of a value
1144 /// you want to see grow smaller, so a change larger than `noise` in the
1145 /// positive direction represents a regression.
1147 /// If `noise` is negative, then it means this metric is of a value
1148 /// you want to see grow larger, so a change larger than `noise` in the
1149 /// negative direction represents a regression.
1150 pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) {
1155 let MetricMap(ref mut map) = *self;
1156 map.insert(name.to_owned(), m);
1159 pub fn fmt_metrics(&self) -> String {
1160 let MetricMap(ref mm) = *self;
1161 let v: Vec<String> = mm.iter()
1162 .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
1171 /// A function that is opaque to the optimizer, to allow benchmarks to
1172 /// pretend to use outputs to assist in avoiding dead-code
1175 /// This function is a no-op, and does not even read from `dummy`.
1176 #[cfg(not(all(target_os = "nacl", target_arch = "le32")))]
1177 pub fn black_box<T>(dummy: T) -> T {
1178 // we need to "use" the argument in some way LLVM can't
1180 unsafe { asm!("" : : "r"(&dummy)) }
1183 #[cfg(all(target_os = "nacl", target_arch = "le32"))]
1185 pub fn black_box<T>(dummy: T) -> T {
1191 /// Callback for benchmark functions to run in their body.
1192 pub fn iter<T, F>(&mut self, mut inner: F)
1193 where F: FnMut() -> T
1195 let start = Instant::now();
1196 let k = self.iterations;
1200 self.dur = start.elapsed();
1203 pub fn ns_elapsed(&mut self) -> u64 {
1204 self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64)
1207 pub fn ns_per_iter(&mut self) -> u64 {
1208 if self.iterations == 0 {
1211 self.ns_elapsed() / cmp::max(self.iterations, 1)
1215 pub fn bench_n<F>(&mut self, n: u64, f: F)
1216 where F: FnOnce(&mut Bencher)
1218 self.iterations = n;
1222 // This is a more statistics-driven benchmark algorithm
1223 pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary
1224 where F: FnMut(&mut Bencher)
1226 // Initial bench run to get ballpark figure.
1228 self.bench_n(n, |x| f(x));
1230 // Try to estimate iter count for 1ms falling back to 1m
1231 // iterations if first run took < 1ns.
1232 if self.ns_per_iter() == 0 {
1235 n = 1_000_000 / cmp::max(self.ns_per_iter(), 1);
1237 // if the first run took more than 1ms we don't want to just
1238 // be left doing 0 iterations on every loop. The unfortunate
1239 // side effect of not being able to do as many runs is
1240 // automatically handled by the statistical analysis below
1241 // (i.e. larger error bars).
1246 let mut total_run = Duration::new(0, 0);
1247 let samples: &mut [f64] = &mut [0.0_f64; 50];
1249 let loop_start = Instant::now();
1251 for p in &mut *samples {
1252 self.bench_n(n, |x| f(x));
1253 *p = self.ns_per_iter() as f64;
1256 stats::winsorize(samples, 5.0);
1257 let summ = stats::Summary::new(samples);
1259 for p in &mut *samples {
1260 self.bench_n(5 * n, |x| f(x));
1261 *p = self.ns_per_iter() as f64;
1264 stats::winsorize(samples, 5.0);
1265 let summ5 = stats::Summary::new(samples);
1266 let loop_run = loop_start.elapsed();
1268 // If we've run for 100ms and seem to have converged to a
1270 if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
1271 summ.median - summ5.median < summ5.median_abs_dev {
1275 total_run = total_run + loop_run;
1276 // Longest we ever run for is 3s.
1277 if total_run > Duration::from_secs(3) {
1281 // If we overflow here just return the results so far. We check a
1282 // multiplier of 10 because we're about to multiply by 2 and the
1283 // next iteration of the loop will also multiply by 5 (to calculate
1284 // the summ5 result)
1285 n = match n.checked_mul(10) {
1287 None => return summ5,
1295 use std::time::Duration;
1296 use super::{Bencher, BenchSamples};
1298 pub fn benchmark<F>(f: F) -> BenchSamples
1299 where F: FnMut(&mut Bencher)
1301 let mut bs = Bencher {
1303 dur: Duration::new(0, 0),
1307 let ns_iter_summ = bs.auto_bench(f);
1309 let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
1310 let mb_s = bs.bytes * 1000 / ns_iter;
1313 ns_iter_summ: ns_iter_summ,
1314 mb_s: mb_s as usize,
1318 pub fn run_once<F>(f: F)
1319 where F: FnOnce(&mut Bencher)
1321 let mut bs = Bencher {
1323 dur: Duration::new(0, 0),
1332 use test::{TrFailed, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc, TestDescAndFn,
1333 TestOpts, run_test, MetricMap, StaticTestName, DynTestName, DynTestFn, ShouldPanic};
1334 use std::sync::mpsc::channel;
1337 pub fn do_not_run_ignored_tests() {
1341 let desc = TestDescAndFn {
1343 name: StaticTestName("whatever"),
1345 should_panic: ShouldPanic::No,
1347 testfn: DynTestFn(Box::new(move || f())),
1349 let (tx, rx) = channel();
1350 run_test(&TestOpts::new(), false, desc, tx);
1351 let (_, res, _) = rx.recv().unwrap();
1352 assert!(res != TrOk);
1356 pub fn ignored_tests_result_in_ignored() {
1358 let desc = TestDescAndFn {
1360 name: StaticTestName("whatever"),
1362 should_panic: ShouldPanic::No,
1364 testfn: DynTestFn(Box::new(move || f())),
1366 let (tx, rx) = channel();
1367 run_test(&TestOpts::new(), false, desc, tx);
1368 let (_, res, _) = rx.recv().unwrap();
1369 assert!(res == TrIgnored);
1373 fn test_should_panic() {
1377 let desc = TestDescAndFn {
1379 name: StaticTestName("whatever"),
1381 should_panic: ShouldPanic::Yes,
1383 testfn: DynTestFn(Box::new(move || f())),
1385 let (tx, rx) = channel();
1386 run_test(&TestOpts::new(), false, desc, tx);
1387 let (_, res, _) = rx.recv().unwrap();
1388 assert!(res == TrOk);
1392 fn test_should_panic_good_message() {
1394 panic!("an error message");
1396 let desc = TestDescAndFn {
1398 name: StaticTestName("whatever"),
1400 should_panic: ShouldPanic::YesWithMessage("error message"),
1402 testfn: DynTestFn(Box::new(move || f())),
1404 let (tx, rx) = channel();
1405 run_test(&TestOpts::new(), false, desc, tx);
1406 let (_, res, _) = rx.recv().unwrap();
1407 assert!(res == TrOk);
1411 fn test_should_panic_bad_message() {
1413 panic!("an error message");
1415 let desc = TestDescAndFn {
1417 name: StaticTestName("whatever"),
1419 should_panic: ShouldPanic::YesWithMessage("foobar"),
1421 testfn: DynTestFn(Box::new(move || f())),
1423 let (tx, rx) = channel();
1424 run_test(&TestOpts::new(), false, desc, tx);
1425 let (_, res, _) = rx.recv().unwrap();
1426 assert!(res == TrFailed);
1430 fn test_should_panic_but_succeeds() {
1432 let desc = TestDescAndFn {
1434 name: StaticTestName("whatever"),
1436 should_panic: ShouldPanic::Yes,
1438 testfn: DynTestFn(Box::new(move || f())),
1440 let (tx, rx) = channel();
1441 run_test(&TestOpts::new(), false, desc, tx);
1442 let (_, res, _) = rx.recv().unwrap();
1443 assert!(res == TrFailed);
1447 fn parse_ignored_flag() {
1448 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
1449 let opts = match parse_opts(&args) {
1451 _ => panic!("Malformed arg in parse_ignored_flag"),
1453 assert!((opts.run_ignored));
1457 pub fn filter_for_ignored_option() {
1458 // When we run ignored tests the test filter should filter out all the
1459 // unignored tests and flip the ignore flag on the rest to false
1461 let mut opts = TestOpts::new();
1462 opts.run_tests = true;
1463 opts.run_ignored = true;
1465 let tests = vec![TestDescAndFn {
1467 name: StaticTestName("1"),
1469 should_panic: ShouldPanic::No,
1471 testfn: DynTestFn(Box::new(move || {})),
1475 name: StaticTestName("2"),
1477 should_panic: ShouldPanic::No,
1479 testfn: DynTestFn(Box::new(move || {})),
1481 let filtered = filter_tests(&opts, tests);
1483 assert_eq!(filtered.len(), 1);
1484 assert_eq!(filtered[0].desc.name.to_string(), "1");
1485 assert!(filtered[0].desc.ignore == false);
1489 pub fn sort_tests() {
1490 let mut opts = TestOpts::new();
1491 opts.run_tests = true;
1493 let names = vec!["sha1::test".to_string(),
1494 "isize::test_to_str".to_string(),
1495 "isize::test_pow".to_string(),
1496 "test::do_not_run_ignored_tests".to_string(),
1497 "test::ignored_tests_result_in_ignored".to_string(),
1498 "test::first_free_arg_should_be_a_filter".to_string(),
1499 "test::parse_ignored_flag".to_string(),
1500 "test::filter_for_ignored_option".to_string(),
1501 "test::sort_tests".to_string()];
1504 let mut tests = Vec::new();
1505 for name in &names {
1506 let test = TestDescAndFn {
1508 name: DynTestName((*name).clone()),
1510 should_panic: ShouldPanic::No,
1512 testfn: DynTestFn(Box::new(testfn)),
1518 let filtered = filter_tests(&opts, tests);
1520 let expected = vec!["isize::test_pow".to_string(),
1521 "isize::test_to_str".to_string(),
1522 "sha1::test".to_string(),
1523 "test::do_not_run_ignored_tests".to_string(),
1524 "test::filter_for_ignored_option".to_string(),
1525 "test::first_free_arg_should_be_a_filter".to_string(),
1526 "test::ignored_tests_result_in_ignored".to_string(),
1527 "test::parse_ignored_flag".to_string(),
1528 "test::sort_tests".to_string()];
1530 for (a, b) in expected.iter().zip(filtered) {
1531 assert!(*a == b.desc.name.to_string());
1536 pub fn test_metricmap_compare() {
1537 let mut m1 = MetricMap::new();
1538 let mut m2 = MetricMap::new();
1539 m1.insert_metric("in-both-noise", 1000.0, 200.0);
1540 m2.insert_metric("in-both-noise", 1100.0, 200.0);
1542 m1.insert_metric("in-first-noise", 1000.0, 2.0);
1543 m2.insert_metric("in-second-noise", 1000.0, 2.0);
1545 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
1546 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
1548 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
1549 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
1551 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
1552 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
1554 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
1555 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);