1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Support code for rustc's built in unit-test and micro-benchmarking
14 //! Almost all user code will only be interested in `Bencher` and
15 //! `black_box`. All other interactions (such as writing tests and
16 //! benchmarks themselves) should be done via the `#[test]` and
17 //! `#[bench]` attributes.
19 //! See the [Testing Chapter](../book/testing.html) of the book for more details.
21 // Currently, not much of this is meant for users. It is intended to
22 // support the simplest interface possible for representing and
23 // running tests while providing a base that other test frameworks may
26 #![crate_name = "test"]
27 #![unstable(feature = "test", issue = "27812")]
28 #![crate_type = "rlib"]
29 #![crate_type = "dylib"]
30 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
31 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
32 html_root_url = "https://doc.rust-lang.org/nightly/",
33 test(attr(deny(warnings))))]
34 #![cfg_attr(not(stage0), deny(warnings))]
37 #![feature(box_syntax)]
40 #![feature(rustc_private)]
41 #![feature(set_stdio)]
42 #![feature(staged_api)]
46 extern crate serialize;
47 extern crate serialize as rustc_serialize;
51 pub use self::TestFn::*;
52 pub use self::ColorConfig::*;
53 pub use self::TestResult::*;
54 pub use self::TestName::*;
55 use self::TestEvent::*;
56 use self::NamePadding::*;
57 use self::OutputLocation::*;
60 use serialize::Encodable;
61 use std::boxed::FnBox;
66 use std::collections::BTreeMap;
70 use std::io::prelude::*;
72 use std::iter::repeat;
73 use std::path::PathBuf;
74 use std::sync::mpsc::{channel, Sender};
75 use std::sync::{Arc, Mutex};
77 use std::time::{Instant, Duration};
79 // to be used by rustc to compile tests in libtest
81 pub use {Bencher, TestName, TestResult, TestDesc, TestDescAndFn, TestOpts, TrFailed,
82 TrIgnored, TrOk, Metric, MetricMap, StaticTestFn, StaticTestName, DynTestName,
83 DynTestFn, run_test, test_main, test_main_static, filter_tests, parse_opts,
84 StaticBenchFn, ShouldPanic};
89 // The name of a test. By convention this follows the rules for rust
90 // paths; i.e. it should be a series of identifiers separated by double
91 // colons. This way if some test runner wants to arrange the tests
92 // hierarchically it may.
94 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
96 StaticTestName(&'static str),
100 fn as_slice(&self) -> &str {
102 StaticTestName(s) => s,
103 DynTestName(ref s) => s,
107 impl fmt::Display for TestName {
108 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
109 fmt::Display::fmt(self.as_slice(), f)
113 #[derive(Clone, Copy)]
120 fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
121 let mut name = String::from(self.name.as_slice());
122 let fill = column_count.saturating_sub(name.len());
123 let pad = repeat(" ").take(fill).collect::<String>();
134 /// Represents a benchmark function.
135 pub trait TDynBenchFn: Send {
136 fn run(&self, harness: &mut Bencher);
139 // A function that runs a test. If the function returns successfully,
140 // the test succeeds; if the function panics then the test fails. We
141 // may need to come up with a more clever definition of test in order
142 // to support isolation of tests into threads.
145 StaticBenchFn(fn(&mut Bencher)),
146 StaticMetricFn(fn(&mut MetricMap)),
147 DynTestFn(Box<FnBox() + Send>),
148 DynMetricFn(Box<FnBox(&mut MetricMap) + Send>),
149 DynBenchFn(Box<TDynBenchFn + 'static>),
153 fn padding(&self) -> NamePadding {
155 StaticTestFn(..) => PadNone,
156 StaticBenchFn(..) => PadOnRight,
157 StaticMetricFn(..) => PadOnRight,
158 DynTestFn(..) => PadNone,
159 DynMetricFn(..) => PadOnRight,
160 DynBenchFn(..) => PadOnRight,
165 impl fmt::Debug for TestFn {
166 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
167 f.write_str(match *self {
168 StaticTestFn(..) => "StaticTestFn(..)",
169 StaticBenchFn(..) => "StaticBenchFn(..)",
170 StaticMetricFn(..) => "StaticMetricFn(..)",
171 DynTestFn(..) => "DynTestFn(..)",
172 DynMetricFn(..) => "DynMetricFn(..)",
173 DynBenchFn(..) => "DynBenchFn(..)",
178 /// Manager of the benchmarking runs.
180 /// This is fed into functions marked with `#[bench]` to allow for
181 /// set-up & tear-down before running a piece of code repeatedly via a
183 #[derive(Copy, Clone)]
190 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
191 pub enum ShouldPanic {
194 YesWithMessage(&'static str),
197 // The definition of a single test. A test runner will run a list of
199 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
200 pub struct TestDesc {
203 pub should_panic: ShouldPanic,
206 unsafe impl Send for TestDesc {}
209 pub struct TestDescAndFn {
214 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug, Copy)]
221 pub fn new(value: f64, noise: f64) -> Metric {
230 pub struct MetricMap(BTreeMap<String, Metric>);
232 impl Clone for MetricMap {
233 fn clone(&self) -> MetricMap {
234 let MetricMap(ref map) = *self;
235 MetricMap(map.clone())
239 // The default console test runner. It accepts the command line
240 // arguments and a vector of test_descs.
241 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>) {
242 let opts = match parse_opts(args) {
244 Some(Err(msg)) => panic!("{:?}", msg),
247 match run_tests_console(&opts, tests) {
249 Ok(false) => std::process::exit(101),
250 Err(e) => panic!("io error when running tests: {:?}", e),
254 // A variant optimized for invocation with a static test vector.
255 // This will panic (intentionally) when fed any dynamic tests, because
256 // it is copying the static values out into a dynamic vector and cannot
257 // copy dynamic values. It is doing this because from this point on
258 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
259 // semantics into parallel test runners, which in turn requires a Vec<>
260 // rather than a &[].
261 pub fn test_main_static(tests: &[TestDescAndFn]) {
262 let args = env::args().collect::<Vec<_>>();
263 let owned_tests = tests.iter()
268 testfn: StaticTestFn(f),
269 desc: t.desc.clone(),
272 StaticBenchFn(f) => {
274 testfn: StaticBenchFn(f),
275 desc: t.desc.clone(),
278 _ => panic!("non-static tests passed to test::test_main_static"),
282 test_main(&args, owned_tests)
285 #[derive(Copy, Clone)]
286 pub enum ColorConfig {
292 pub struct TestOpts {
293 pub filter: Option<String>,
294 pub run_ignored: bool,
296 pub bench_benchmarks: bool,
297 pub logfile: Option<PathBuf>,
299 pub color: ColorConfig,
304 fn new() -> TestOpts {
309 bench_benchmarks: false,
317 /// Result of parsing the options.
318 pub type OptRes = Result<TestOpts, String>;
320 #[cfg_attr(rustfmt, rustfmt_skip)]
321 fn optgroups() -> Vec<getopts::OptGroup> {
322 vec!(getopts::optflag("", "ignored", "Run ignored tests"),
323 getopts::optflag("", "test", "Run tests and not benchmarks"),
324 getopts::optflag("", "bench", "Run benchmarks instead of tests"),
325 getopts::optflag("h", "help", "Display this message (longer with --help)"),
326 getopts::optopt("", "logfile", "Write logs to the specified file instead \
328 getopts::optflag("", "nocapture", "don't capture stdout/stderr of each \
329 task, allow printing directly"),
330 getopts::optopt("", "color", "Configure coloring of output:
331 auto = colorize if stdout is a tty and tests are run on serially (default);
332 always = always colorize output;
333 never = never colorize output;", "auto|always|never"))
336 fn usage(binary: &str) {
337 let message = format!("Usage: {} [OPTIONS] [FILTER]", binary);
340 The FILTER string is tested against the name of all tests, and only those
341 tests whose names contain the filter are run.
343 By default, all tests are run in parallel. This can be altered with the
344 RUST_TEST_THREADS environment variable when running tests (set it to 1).
346 All tests have their standard output and standard error captured by default.
347 This can be overridden with the --nocapture flag or the RUST_TEST_NOCAPTURE=1
348 environment variable. Logging is not captured by default.
352 #[test] - Indicates a function is a test to be run. This function
354 #[bench] - Indicates a function is a benchmark to be run. This
355 function takes one argument (test::Bencher).
356 #[should_panic] - This function (also labeled with #[test]) will only pass if
357 the code causes a panic (an assertion failure or panic!)
358 A message may be provided, which the failure string must
359 contain: #[should_panic(expected = "foo")].
360 #[ignore] - When applied to a function which is already attributed as a
361 test, then the test runner will ignore these tests during
362 normal test runs. Running with --ignored will run these
364 usage = getopts::usage(&message, &optgroups()));
367 // Parses command line arguments into test options
368 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
369 let args_ = &args[1..];
370 let matches = match getopts::getopts(args_, &optgroups()) {
372 Err(f) => return Some(Err(f.to_string())),
375 if matches.opt_present("h") {
380 let filter = if !matches.free.is_empty() {
381 Some(matches.free[0].clone())
386 let run_ignored = matches.opt_present("ignored");
388 let logfile = matches.opt_str("logfile");
389 let logfile = logfile.map(|s| PathBuf::from(&s));
391 let bench_benchmarks = matches.opt_present("bench");
392 let run_tests = !bench_benchmarks || matches.opt_present("test");
394 let mut nocapture = matches.opt_present("nocapture");
396 nocapture = env::var("RUST_TEST_NOCAPTURE").is_ok();
399 let color = match matches.opt_str("color").as_ref().map(|s| &**s) {
400 Some("auto") | None => AutoColor,
401 Some("always") => AlwaysColor,
402 Some("never") => NeverColor,
405 return Some(Err(format!("argument for --color must be auto, always, or never (was \
411 let test_opts = TestOpts {
413 run_ignored: run_ignored,
414 run_tests: run_tests,
415 bench_benchmarks: bench_benchmarks,
417 nocapture: nocapture,
424 #[derive(Clone, PartialEq)]
425 pub struct BenchSamples {
426 ns_iter_summ: stats::Summary,
430 #[derive(Clone, PartialEq)]
431 pub enum TestResult {
435 TrMetrics(MetricMap),
436 TrBench(BenchSamples),
439 unsafe impl Send for TestResult {}
441 enum OutputLocation<T> {
442 Pretty(Box<term::StdoutTerminal>),
446 struct ConsoleTestState<T> {
447 log_out: Option<File>,
448 out: OutputLocation<T>,
456 failures: Vec<(TestDesc, Vec<u8>)>,
457 max_name_len: usize, // number of columns to fill when aligning names
460 impl<T: Write> ConsoleTestState<T> {
461 pub fn new(opts: &TestOpts, _: Option<T>) -> io::Result<ConsoleTestState<io::Stdout>> {
462 let log_out = match opts.logfile {
463 Some(ref path) => Some(try!(File::create(path))),
466 let out = match term::stdout() {
467 None => Raw(io::stdout()),
468 Some(t) => Pretty(t),
471 Ok(ConsoleTestState {
474 use_color: use_color(opts),
480 metrics: MetricMap::new(),
481 failures: Vec::new(),
486 pub fn write_ok(&mut self) -> io::Result<()> {
487 self.write_pretty("ok", term::color::GREEN)
490 pub fn write_failed(&mut self) -> io::Result<()> {
491 self.write_pretty("FAILED", term::color::RED)
494 pub fn write_ignored(&mut self) -> io::Result<()> {
495 self.write_pretty("ignored", term::color::YELLOW)
498 pub fn write_metric(&mut self) -> io::Result<()> {
499 self.write_pretty("metric", term::color::CYAN)
502 pub fn write_bench(&mut self) -> io::Result<()> {
503 self.write_pretty("bench", term::color::CYAN)
506 pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
508 Pretty(ref mut term) => {
510 try!(term.fg(color));
512 try!(term.write_all(word.as_bytes()));
518 Raw(ref mut stdout) => {
519 try!(stdout.write_all(word.as_bytes()));
525 pub fn write_plain(&mut self, s: &str) -> io::Result<()> {
527 Pretty(ref mut term) => {
528 try!(term.write_all(s.as_bytes()));
531 Raw(ref mut stdout) => {
532 try!(stdout.write_all(s.as_bytes()));
538 pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
540 let noun = if len != 1 {
545 self.write_plain(&format!("\nrunning {} {}\n", len, noun))
548 pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
549 let name = test.padded_name(self.max_name_len, align);
550 self.write_plain(&format!("test {} ... ", name))
553 pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
555 TrOk => self.write_ok(),
556 TrFailed => self.write_failed(),
557 TrIgnored => self.write_ignored(),
558 TrMetrics(ref mm) => {
559 try!(self.write_metric());
560 self.write_plain(&format!(": {}", mm.fmt_metrics()))
563 try!(self.write_bench());
565 try!(self.write_plain(&format!(": {}", fmt_bench_samples(bs))));
570 self.write_plain("\n")
573 pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
577 let s = format!("{} {}\n",
579 TrOk => "ok".to_owned(),
580 TrFailed => "failed".to_owned(),
581 TrIgnored => "ignored".to_owned(),
582 TrMetrics(ref mm) => mm.fmt_metrics(),
583 TrBench(ref bs) => fmt_bench_samples(bs),
586 o.write_all(s.as_bytes())
591 pub fn write_failures(&mut self) -> io::Result<()> {
592 try!(self.write_plain("\nfailures:\n"));
593 let mut failures = Vec::new();
594 let mut fail_out = String::new();
595 for &(ref f, ref stdout) in &self.failures {
596 failures.push(f.name.to_string());
597 if !stdout.is_empty() {
598 fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
599 let output = String::from_utf8_lossy(stdout);
600 fail_out.push_str(&output);
601 fail_out.push_str("\n");
604 if !fail_out.is_empty() {
605 try!(self.write_plain("\n"));
606 try!(self.write_plain(&fail_out));
609 try!(self.write_plain("\nfailures:\n"));
611 for name in &failures {
612 try!(self.write_plain(&format!(" {}\n", name)));
617 pub fn write_run_finish(&mut self) -> io::Result<bool> {
618 assert!(self.passed + self.failed + self.ignored + self.measured == self.total);
620 let success = self.failed == 0;
622 try!(self.write_failures());
625 try!(self.write_plain("\ntest result: "));
627 // There's no parallelism at this point so it's safe to use color
628 try!(self.write_ok());
630 try!(self.write_failed());
632 let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n",
637 try!(self.write_plain(&s));
642 // Format a number with thousands separators
643 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
645 let mut output = String::new();
646 let mut trailing = false;
647 for &pow in &[9, 6, 3, 0] {
648 let base = 10_usize.pow(pow);
649 if pow == 0 || trailing || n / base != 0 {
651 output.write_fmt(format_args!("{}", n / base)).unwrap();
653 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
666 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
668 let mut output = String::new();
670 let median = bs.ns_iter_summ.median as usize;
671 let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
673 output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
674 fmt_thousands_sep(median, ','),
675 fmt_thousands_sep(deviation, ',')))
678 output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
683 // A simple console test runner
684 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
686 fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
687 match (*event).clone() {
688 TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
689 TeWait(ref test, padding) => st.write_test_start(test, padding),
690 TeResult(test, result, stdout) => {
691 try!(st.write_log(&test, &result));
692 try!(st.write_result(&result));
694 TrOk => st.passed += 1,
695 TrIgnored => st.ignored += 1,
697 let tname = test.name;
698 let MetricMap(mm) = mm;
701 .insert_metric(&format!("{}.{}", tname, k), v.value, v.noise);
706 st.metrics.insert_metric(test.name.as_slice(),
707 bs.ns_iter_summ.median,
708 bs.ns_iter_summ.max - bs.ns_iter_summ.min);
713 st.failures.push((test, stdout));
721 let mut st = try!(ConsoleTestState::new(opts, None::<io::Stdout>));
722 fn len_if_padded(t: &TestDescAndFn) -> usize {
723 match t.testfn.padding() {
725 PadOnRight => t.desc.name.as_slice().len(),
728 match tests.iter().max_by_key(|t| len_if_padded(*t)) {
730 let n = t.desc.name.as_slice();
731 st.max_name_len = n.len();
735 try!(run_tests(opts, tests, |x| callback(&x, &mut st)));
736 return st.write_run_finish();
740 fn should_sort_failures_before_printing_them() {
741 let test_a = TestDesc {
742 name: StaticTestName("a"),
744 should_panic: ShouldPanic::No,
747 let test_b = TestDesc {
748 name: StaticTestName("b"),
750 should_panic: ShouldPanic::No,
753 let mut st = ConsoleTestState {
755 out: Raw(Vec::new()),
763 metrics: MetricMap::new(),
764 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
767 st.write_failures().unwrap();
768 let s = match st.out {
769 Raw(ref m) => String::from_utf8_lossy(&m[..]),
770 Pretty(_) => unreachable!(),
773 let apos = s.find("a").unwrap();
774 let bpos = s.find("b").unwrap();
775 assert!(apos < bpos);
778 fn use_color(opts: &TestOpts) -> bool {
780 AutoColor => !opts.nocapture && stdout_isatty(),
787 fn stdout_isatty() -> bool {
788 unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
791 fn stdout_isatty() -> bool {
794 type HANDLE = *mut u8;
795 type LPDWORD = *mut u32;
796 const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD;
798 fn GetStdHandle(which: DWORD) -> HANDLE;
799 fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL;
802 let handle = GetStdHandle(STD_OUTPUT_HANDLE);
804 GetConsoleMode(handle, &mut out) != 0
810 TeFiltered(Vec<TestDesc>),
811 TeWait(TestDesc, NamePadding),
812 TeResult(TestDesc, TestResult, Vec<u8>),
815 pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
818 fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
819 where F: FnMut(TestEvent) -> io::Result<()>
821 let mut filtered_tests = filter_tests(opts, tests);
822 if !opts.bench_benchmarks {
823 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
826 let filtered_descs = filtered_tests.iter()
827 .map(|t| t.desc.clone())
830 try!(callback(TeFiltered(filtered_descs)));
832 let (filtered_tests, filtered_benchs_and_metrics): (Vec<_>, _) =
833 filtered_tests.into_iter().partition(|e| {
835 StaticTestFn(_) | DynTestFn(_) => true,
840 // It's tempting to just spawn all the tests at once, but since we have
841 // many tests that run in other processes we would be making a big mess.
842 let concurrency = get_concurrency();
844 let mut remaining = filtered_tests;
848 let (tx, rx) = channel::<MonitorMsg>();
850 while pending > 0 || !remaining.is_empty() {
851 while pending < concurrency && !remaining.is_empty() {
852 let test = remaining.pop().unwrap();
853 if concurrency == 1 {
854 // We are doing one test at a time so we can print the name
855 // of the test before we run it. Useful for debugging tests
856 // that hang forever.
857 try!(callback(TeWait(test.desc.clone(), test.testfn.padding())));
859 run_test(opts, !opts.run_tests, test, tx.clone());
863 let (desc, result, stdout) = rx.recv().unwrap();
864 if concurrency != 1 {
865 try!(callback(TeWait(desc.clone(), PadNone)));
867 try!(callback(TeResult(desc, result, stdout)));
871 if opts.bench_benchmarks {
872 // All benchmarks run at the end, in serial.
873 // (this includes metric fns)
874 for b in filtered_benchs_and_metrics {
875 try!(callback(TeWait(b.desc.clone(), b.testfn.padding())));
876 run_test(opts, false, b, tx.clone());
877 let (test, result, stdout) = rx.recv().unwrap();
878 try!(callback(TeResult(test, result, stdout)));
885 fn get_concurrency() -> usize {
886 return match env::var("RUST_TEST_THREADS") {
888 let opt_n: Option<usize> = s.parse().ok();
890 Some(n) if n > 0 => n,
892 panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.",
897 Err(..) => num_cpus(),
902 fn num_cpus() -> usize {
905 wProcessorArchitecture: u16,
908 lpMinimumApplicationAddress: *mut u8,
909 lpMaximumApplicationAddress: *mut u8,
910 dwActiveProcessorMask: *mut u8,
911 dwNumberOfProcessors: u32,
912 dwProcessorType: u32,
913 dwAllocationGranularity: u32,
914 wProcessorLevel: u16,
915 wProcessorRevision: u16,
918 fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32;
921 let mut sysinfo = std::mem::zeroed();
922 GetSystemInfo(&mut sysinfo);
923 sysinfo.dwNumberOfProcessors as usize
927 #[cfg(any(target_os = "linux",
930 target_os = "android",
931 target_os = "solaris"))]
932 fn num_cpus() -> usize {
933 unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
936 #[cfg(any(target_os = "freebsd",
937 target_os = "dragonfly",
938 target_os = "bitrig",
939 target_os = "netbsd"))]
940 fn num_cpus() -> usize {
941 let mut cpus: libc::c_uint = 0;
942 let mut cpus_size = std::mem::size_of_val(&cpus);
945 cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint;
948 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
950 libc::sysctl(mib.as_mut_ptr(),
952 &mut cpus as *mut _ as *mut _,
953 &mut cpus_size as *mut _ as *mut _,
964 #[cfg(target_os = "openbsd")]
965 fn num_cpus() -> usize {
966 let mut cpus: libc::c_uint = 0;
967 let mut cpus_size = std::mem::size_of_val(&cpus);
968 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
971 libc::sysctl(mib.as_mut_ptr(),
973 &mut cpus as *mut _ as *mut _,
974 &mut cpus_size as *mut _ as *mut _,
985 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
986 let mut filtered = tests;
988 // Remove tests that don't match the test filter
989 filtered = match opts.filter {
991 Some(ref filter) => {
993 .filter(|test| test.desc.name.as_slice().contains(&filter[..]))
998 // Maybe pull out the ignored test and unignore them
999 filtered = if !opts.run_ignored {
1002 fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
1003 if test.desc.ignore {
1004 let TestDescAndFn {desc, testfn} = test;
1005 Some(TestDescAndFn {
1006 desc: TestDesc { ignore: false, ..desc },
1013 filtered.into_iter().filter_map(filter).collect()
1016 // Sort the tests alphabetically
1017 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
1022 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1023 // convert benchmarks to tests, if we're not benchmarking them
1026 let testfn = match x.testfn {
1027 DynBenchFn(bench) => {
1028 DynTestFn(Box::new(move || bench::run_once(|b| bench.run(b))))
1030 StaticBenchFn(benchfn) => {
1031 DynTestFn(Box::new(move || bench::run_once(|b| benchfn(b))))
1043 pub fn run_test(opts: &TestOpts,
1045 test: TestDescAndFn,
1046 monitor_ch: Sender<MonitorMsg>) {
1048 let TestDescAndFn {desc, testfn} = test;
1050 if force_ignore || desc.ignore {
1051 monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap();
1055 fn run_test_inner(desc: TestDesc,
1056 monitor_ch: Sender<MonitorMsg>,
1058 testfn: Box<FnBox() + Send>) {
1059 struct Sink(Arc<Mutex<Vec<u8>>>);
1060 impl Write for Sink {
1061 fn write(&mut self, data: &[u8]) -> io::Result<usize> {
1062 Write::write(&mut *self.0.lock().unwrap(), data)
1064 fn flush(&mut self) -> io::Result<()> {
1069 thread::spawn(move || {
1070 let data = Arc::new(Mutex::new(Vec::new()));
1071 let data2 = data.clone();
1072 let cfg = thread::Builder::new().name(match desc.name {
1073 DynTestName(ref name) => name.clone(),
1074 StaticTestName(name) => name.to_owned(),
1077 let result_guard = cfg.spawn(move || {
1079 io::set_print(box Sink(data2.clone()));
1080 io::set_panic(box Sink(data2));
1085 let test_result = calc_result(&desc, result_guard.join());
1086 let stdout = data.lock().unwrap().to_vec();
1087 monitor_ch.send((desc.clone(), test_result, stdout)).unwrap();
1092 DynBenchFn(bencher) => {
1093 let bs = ::bench::benchmark(|harness| bencher.run(harness));
1094 monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
1097 StaticBenchFn(benchfn) => {
1098 let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness));
1099 monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
1103 let mut mm = MetricMap::new();
1104 f.call_box((&mut mm,));
1105 monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap();
1108 StaticMetricFn(f) => {
1109 let mut mm = MetricMap::new();
1111 monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap();
1114 DynTestFn(f) => run_test_inner(desc, monitor_ch, opts.nocapture, f),
1115 StaticTestFn(f) => run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(f)),
1119 fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> TestResult {
1120 match (&desc.should_panic, task_result) {
1121 (&ShouldPanic::No, Ok(())) |
1122 (&ShouldPanic::Yes, Err(_)) => TrOk,
1123 (&ShouldPanic::YesWithMessage(msg), Err(ref err))
1124 if err.downcast_ref::<String>()
1126 .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
1127 .map(|e| e.contains(msg))
1128 .unwrap_or(false) => TrOk,
1134 pub fn new() -> MetricMap {
1135 MetricMap(BTreeMap::new())
1138 /// Insert a named `value` (+/- `noise`) metric into the map. The value
1139 /// must be non-negative. The `noise` indicates the uncertainty of the
1140 /// metric, which doubles as the "noise range" of acceptable
1141 /// pairwise-regressions on this named value, when comparing from one
1142 /// metric to the next using `compare_to_old`.
1144 /// If `noise` is positive, then it means this metric is of a value
1145 /// you want to see grow smaller, so a change larger than `noise` in the
1146 /// positive direction represents a regression.
1148 /// If `noise` is negative, then it means this metric is of a value
1149 /// you want to see grow larger, so a change larger than `noise` in the
1150 /// negative direction represents a regression.
1151 pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) {
1156 let MetricMap(ref mut map) = *self;
1157 map.insert(name.to_owned(), m);
1160 pub fn fmt_metrics(&self) -> String {
1161 let MetricMap(ref mm) = *self;
1162 let v: Vec<String> = mm.iter()
1163 .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
1172 /// A function that is opaque to the optimizer, to allow benchmarks to
1173 /// pretend to use outputs to assist in avoiding dead-code
1176 /// This function is a no-op, and does not even read from `dummy`.
1177 #[cfg(not(all(target_os = "nacl", target_arch = "le32")))]
1178 pub fn black_box<T>(dummy: T) -> T {
1179 // we need to "use" the argument in some way LLVM can't
1181 unsafe { asm!("" : : "r"(&dummy)) }
1184 #[cfg(all(target_os = "nacl", target_arch = "le32"))]
1186 pub fn black_box<T>(dummy: T) -> T {
1192 /// Callback for benchmark functions to run in their body.
1193 pub fn iter<T, F>(&mut self, mut inner: F)
1194 where F: FnMut() -> T
1196 let start = Instant::now();
1197 let k = self.iterations;
1201 self.dur = start.elapsed();
1204 pub fn ns_elapsed(&mut self) -> u64 {
1205 self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64)
1208 pub fn ns_per_iter(&mut self) -> u64 {
1209 if self.iterations == 0 {
1212 self.ns_elapsed() / cmp::max(self.iterations, 1)
1216 pub fn bench_n<F>(&mut self, n: u64, f: F)
1217 where F: FnOnce(&mut Bencher)
1219 self.iterations = n;
1223 // This is a more statistics-driven benchmark algorithm
1224 pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary
1225 where F: FnMut(&mut Bencher)
1227 // Initial bench run to get ballpark figure.
1229 self.bench_n(n, |x| f(x));
1231 // Try to estimate iter count for 1ms falling back to 1m
1232 // iterations if first run took < 1ns.
1233 if self.ns_per_iter() == 0 {
1236 n = 1_000_000 / cmp::max(self.ns_per_iter(), 1);
1238 // if the first run took more than 1ms we don't want to just
1239 // be left doing 0 iterations on every loop. The unfortunate
1240 // side effect of not being able to do as many runs is
1241 // automatically handled by the statistical analysis below
1242 // (i.e. larger error bars).
1247 let mut total_run = Duration::new(0, 0);
1248 let samples: &mut [f64] = &mut [0.0_f64; 50];
1250 let loop_start = Instant::now();
1252 for p in &mut *samples {
1253 self.bench_n(n, |x| f(x));
1254 *p = self.ns_per_iter() as f64;
1257 stats::winsorize(samples, 5.0);
1258 let summ = stats::Summary::new(samples);
1260 for p in &mut *samples {
1261 self.bench_n(5 * n, |x| f(x));
1262 *p = self.ns_per_iter() as f64;
1265 stats::winsorize(samples, 5.0);
1266 let summ5 = stats::Summary::new(samples);
1267 let loop_run = loop_start.elapsed();
1269 // If we've run for 100ms and seem to have converged to a
1271 if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
1272 summ.median - summ5.median < summ5.median_abs_dev {
1276 total_run = total_run + loop_run;
1277 // Longest we ever run for is 3s.
1278 if total_run > Duration::from_secs(3) {
1282 // If we overflow here just return the results so far. We check a
1283 // multiplier of 10 because we're about to multiply by 2 and the
1284 // next iteration of the loop will also multiply by 5 (to calculate
1285 // the summ5 result)
1286 n = match n.checked_mul(10) {
1288 None => return summ5,
1296 use std::time::Duration;
1297 use super::{Bencher, BenchSamples};
1299 pub fn benchmark<F>(f: F) -> BenchSamples
1300 where F: FnMut(&mut Bencher)
1302 let mut bs = Bencher {
1304 dur: Duration::new(0, 0),
1308 let ns_iter_summ = bs.auto_bench(f);
1310 let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
1311 let mb_s = bs.bytes * 1000 / ns_iter;
1314 ns_iter_summ: ns_iter_summ,
1315 mb_s: mb_s as usize,
1319 pub fn run_once<F>(f: F)
1320 where F: FnOnce(&mut Bencher)
1322 let mut bs = Bencher {
1324 dur: Duration::new(0, 0),
1333 use test::{TrFailed, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc, TestDescAndFn,
1334 TestOpts, run_test, MetricMap, StaticTestName, DynTestName, DynTestFn, ShouldPanic};
1335 use std::sync::mpsc::channel;
1338 pub fn do_not_run_ignored_tests() {
1342 let desc = TestDescAndFn {
1344 name: StaticTestName("whatever"),
1346 should_panic: ShouldPanic::No,
1348 testfn: DynTestFn(Box::new(move || f())),
1350 let (tx, rx) = channel();
1351 run_test(&TestOpts::new(), false, desc, tx);
1352 let (_, res, _) = rx.recv().unwrap();
1353 assert!(res != TrOk);
1357 pub fn ignored_tests_result_in_ignored() {
1359 let desc = TestDescAndFn {
1361 name: StaticTestName("whatever"),
1363 should_panic: ShouldPanic::No,
1365 testfn: DynTestFn(Box::new(move || f())),
1367 let (tx, rx) = channel();
1368 run_test(&TestOpts::new(), false, desc, tx);
1369 let (_, res, _) = rx.recv().unwrap();
1370 assert!(res == TrIgnored);
1374 fn test_should_panic() {
1378 let desc = TestDescAndFn {
1380 name: StaticTestName("whatever"),
1382 should_panic: ShouldPanic::Yes,
1384 testfn: DynTestFn(Box::new(move || f())),
1386 let (tx, rx) = channel();
1387 run_test(&TestOpts::new(), false, desc, tx);
1388 let (_, res, _) = rx.recv().unwrap();
1389 assert!(res == TrOk);
1393 fn test_should_panic_good_message() {
1395 panic!("an error message");
1397 let desc = TestDescAndFn {
1399 name: StaticTestName("whatever"),
1401 should_panic: ShouldPanic::YesWithMessage("error message"),
1403 testfn: DynTestFn(Box::new(move || f())),
1405 let (tx, rx) = channel();
1406 run_test(&TestOpts::new(), false, desc, tx);
1407 let (_, res, _) = rx.recv().unwrap();
1408 assert!(res == TrOk);
1412 fn test_should_panic_bad_message() {
1414 panic!("an error message");
1416 let desc = TestDescAndFn {
1418 name: StaticTestName("whatever"),
1420 should_panic: ShouldPanic::YesWithMessage("foobar"),
1422 testfn: DynTestFn(Box::new(move || f())),
1424 let (tx, rx) = channel();
1425 run_test(&TestOpts::new(), false, desc, tx);
1426 let (_, res, _) = rx.recv().unwrap();
1427 assert!(res == TrFailed);
1431 fn test_should_panic_but_succeeds() {
1433 let desc = TestDescAndFn {
1435 name: StaticTestName("whatever"),
1437 should_panic: ShouldPanic::Yes,
1439 testfn: DynTestFn(Box::new(move || f())),
1441 let (tx, rx) = channel();
1442 run_test(&TestOpts::new(), false, desc, tx);
1443 let (_, res, _) = rx.recv().unwrap();
1444 assert!(res == TrFailed);
1448 fn parse_ignored_flag() {
1449 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
1450 let opts = match parse_opts(&args) {
1452 _ => panic!("Malformed arg in parse_ignored_flag"),
1454 assert!((opts.run_ignored));
1458 pub fn filter_for_ignored_option() {
1459 // When we run ignored tests the test filter should filter out all the
1460 // unignored tests and flip the ignore flag on the rest to false
1462 let mut opts = TestOpts::new();
1463 opts.run_tests = true;
1464 opts.run_ignored = true;
1466 let tests = vec![TestDescAndFn {
1468 name: StaticTestName("1"),
1470 should_panic: ShouldPanic::No,
1472 testfn: DynTestFn(Box::new(move || {})),
1476 name: StaticTestName("2"),
1478 should_panic: ShouldPanic::No,
1480 testfn: DynTestFn(Box::new(move || {})),
1482 let filtered = filter_tests(&opts, tests);
1484 assert_eq!(filtered.len(), 1);
1485 assert_eq!(filtered[0].desc.name.to_string(), "1");
1486 assert!(filtered[0].desc.ignore == false);
1490 pub fn sort_tests() {
1491 let mut opts = TestOpts::new();
1492 opts.run_tests = true;
1494 let names = vec!["sha1::test".to_string(),
1495 "isize::test_to_str".to_string(),
1496 "isize::test_pow".to_string(),
1497 "test::do_not_run_ignored_tests".to_string(),
1498 "test::ignored_tests_result_in_ignored".to_string(),
1499 "test::first_free_arg_should_be_a_filter".to_string(),
1500 "test::parse_ignored_flag".to_string(),
1501 "test::filter_for_ignored_option".to_string(),
1502 "test::sort_tests".to_string()];
1505 let mut tests = Vec::new();
1506 for name in &names {
1507 let test = TestDescAndFn {
1509 name: DynTestName((*name).clone()),
1511 should_panic: ShouldPanic::No,
1513 testfn: DynTestFn(Box::new(testfn)),
1519 let filtered = filter_tests(&opts, tests);
1521 let expected = vec!["isize::test_pow".to_string(),
1522 "isize::test_to_str".to_string(),
1523 "sha1::test".to_string(),
1524 "test::do_not_run_ignored_tests".to_string(),
1525 "test::filter_for_ignored_option".to_string(),
1526 "test::first_free_arg_should_be_a_filter".to_string(),
1527 "test::ignored_tests_result_in_ignored".to_string(),
1528 "test::parse_ignored_flag".to_string(),
1529 "test::sort_tests".to_string()];
1531 for (a, b) in expected.iter().zip(filtered) {
1532 assert!(*a == b.desc.name.to_string());
1537 pub fn test_metricmap_compare() {
1538 let mut m1 = MetricMap::new();
1539 let mut m2 = MetricMap::new();
1540 m1.insert_metric("in-both-noise", 1000.0, 200.0);
1541 m2.insert_metric("in-both-noise", 1100.0, 200.0);
1543 m1.insert_metric("in-first-noise", 1000.0, 2.0);
1544 m2.insert_metric("in-second-noise", 1000.0, 2.0);
1546 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
1547 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
1549 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
1550 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
1552 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
1553 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
1555 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
1556 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);