1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Support code for rustc's built in unit-test and micro-benchmarking
14 //! Almost all user code will only be interested in `Bencher` and
15 //! `black_box`. All other interactions (such as writing tests and
16 //! benchmarks themselves) should be done via the `#[test]` and
17 //! `#[bench]` attributes.
19 //! See the [Testing Chapter](../book/testing.html) of the book for more details.
21 // Currently, not much of this is meant for users. It is intended to
22 // support the simplest interface possible for representing and
23 // running tests while providing a base that other test frameworks may
26 #![crate_name = "test"]
27 #![unstable(feature = "test", issue = "27812")]
28 #![crate_type = "rlib"]
29 #![crate_type = "dylib"]
30 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
31 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
32 html_root_url = "https://doc.rust-lang.org/nightly/",
33 test(attr(deny(warnings))))]
34 #![cfg_attr(not(stage0), deny(warnings))]
37 #![feature(box_syntax)]
40 #![feature(rustc_private)]
41 #![feature(set_stdio)]
42 #![feature(staged_api)]
46 extern crate serialize;
47 extern crate serialize as rustc_serialize;
51 pub use self::TestFn::*;
52 pub use self::ColorConfig::*;
53 pub use self::TestResult::*;
54 pub use self::TestName::*;
55 use self::TestEvent::*;
56 use self::NamePadding::*;
57 use self::OutputLocation::*;
60 use serialize::Encodable;
61 use std::boxed::FnBox;
66 use std::collections::BTreeMap;
70 use std::io::prelude::*;
72 use std::iter::repeat;
73 use std::path::PathBuf;
74 use std::sync::mpsc::{channel, Sender};
75 use std::sync::{Arc, Mutex};
77 use std::time::{Instant, Duration};
79 // to be used by rustc to compile tests in libtest
81 pub use {Bencher, TestName, TestResult, TestDesc, TestDescAndFn, TestOpts, TrFailed,
82 TrIgnored, TrOk, Metric, MetricMap, StaticTestFn, StaticTestName, DynTestName,
83 DynTestFn, run_test, test_main, test_main_static, filter_tests, parse_opts,
84 StaticBenchFn, ShouldPanic};
89 // The name of a test. By convention this follows the rules for rust
90 // paths; i.e. it should be a series of identifiers separated by double
91 // colons. This way if some test runner wants to arrange the tests
92 // hierarchically it may.
94 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
96 StaticTestName(&'static str),
100 fn as_slice(&self) -> &str {
102 StaticTestName(s) => s,
103 DynTestName(ref s) => s,
107 impl fmt::Display for TestName {
108 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
109 fmt::Display::fmt(self.as_slice(), f)
113 #[derive(Clone, Copy)]
120 fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
121 let mut name = String::from(self.name.as_slice());
122 let fill = column_count.saturating_sub(name.len());
123 let pad = repeat(" ").take(fill).collect::<String>();
134 /// Represents a benchmark function.
135 pub trait TDynBenchFn: Send {
136 fn run(&self, harness: &mut Bencher);
139 // A function that runs a test. If the function returns successfully,
140 // the test succeeds; if the function panics then the test fails. We
141 // may need to come up with a more clever definition of test in order
142 // to support isolation of tests into threads.
145 StaticBenchFn(fn(&mut Bencher)),
146 StaticMetricFn(fn(&mut MetricMap)),
147 DynTestFn(Box<FnBox() + Send>),
148 DynMetricFn(Box<FnBox(&mut MetricMap) + Send>),
149 DynBenchFn(Box<TDynBenchFn + 'static>),
153 fn padding(&self) -> NamePadding {
155 StaticTestFn(..) => PadNone,
156 StaticBenchFn(..) => PadOnRight,
157 StaticMetricFn(..) => PadOnRight,
158 DynTestFn(..) => PadNone,
159 DynMetricFn(..) => PadOnRight,
160 DynBenchFn(..) => PadOnRight,
165 impl fmt::Debug for TestFn {
166 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
167 f.write_str(match *self {
168 StaticTestFn(..) => "StaticTestFn(..)",
169 StaticBenchFn(..) => "StaticBenchFn(..)",
170 StaticMetricFn(..) => "StaticMetricFn(..)",
171 DynTestFn(..) => "DynTestFn(..)",
172 DynMetricFn(..) => "DynMetricFn(..)",
173 DynBenchFn(..) => "DynBenchFn(..)",
178 /// Manager of the benchmarking runs.
180 /// This is fed into functions marked with `#[bench]` to allow for
181 /// set-up & tear-down before running a piece of code repeatedly via a
183 #[derive(Copy, Clone)]
190 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
191 pub enum ShouldPanic {
194 YesWithMessage(&'static str),
197 // The definition of a single test. A test runner will run a list of
199 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
200 pub struct TestDesc {
203 pub should_panic: ShouldPanic,
206 unsafe impl Send for TestDesc {}
209 pub struct TestDescAndFn {
214 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug, Copy)]
221 pub fn new(value: f64, noise: f64) -> Metric {
230 pub struct MetricMap(BTreeMap<String, Metric>);
232 impl Clone for MetricMap {
233 fn clone(&self) -> MetricMap {
234 let MetricMap(ref map) = *self;
235 MetricMap(map.clone())
239 // The default console test runner. It accepts the command line
240 // arguments and a vector of test_descs.
241 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>) {
242 let opts = match parse_opts(args) {
244 Some(Err(msg)) => panic!("{:?}", msg),
247 match run_tests_console(&opts, tests) {
249 Ok(false) => std::process::exit(101),
250 Err(e) => panic!("io error when running tests: {:?}", e),
254 // A variant optimized for invocation with a static test vector.
255 // This will panic (intentionally) when fed any dynamic tests, because
256 // it is copying the static values out into a dynamic vector and cannot
257 // copy dynamic values. It is doing this because from this point on
258 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
259 // semantics into parallel test runners, which in turn requires a Vec<>
260 // rather than a &[].
261 pub fn test_main_static(tests: &[TestDescAndFn]) {
262 let args = env::args().collect::<Vec<_>>();
263 let owned_tests = tests.iter()
268 testfn: StaticTestFn(f),
269 desc: t.desc.clone(),
272 StaticBenchFn(f) => {
274 testfn: StaticBenchFn(f),
275 desc: t.desc.clone(),
278 _ => panic!("non-static tests passed to test::test_main_static"),
282 test_main(&args, owned_tests)
285 #[derive(Copy, Clone)]
286 pub enum ColorConfig {
292 pub struct TestOpts {
293 pub filter: Option<String>,
294 pub run_ignored: bool,
296 pub bench_benchmarks: bool,
297 pub logfile: Option<PathBuf>,
299 pub color: ColorConfig,
304 fn new() -> TestOpts {
309 bench_benchmarks: false,
317 /// Result of parsing the options.
318 pub type OptRes = Result<TestOpts, String>;
320 #[cfg_attr(rustfmt, rustfmt_skip)]
321 fn optgroups() -> Vec<getopts::OptGroup> {
322 vec!(getopts::optflag("", "ignored", "Run ignored tests"),
323 getopts::optflag("", "test", "Run tests and not benchmarks"),
324 getopts::optflag("", "bench", "Run benchmarks instead of tests"),
325 getopts::optflag("h", "help", "Display this message (longer with --help)"),
326 getopts::optopt("", "logfile", "Write logs to the specified file instead \
328 getopts::optflag("", "nocapture", "don't capture stdout/stderr of each \
329 task, allow printing directly"),
330 getopts::optopt("", "color", "Configure coloring of output:
331 auto = colorize if stdout is a tty and tests are run on serially (default);
332 always = always colorize output;
333 never = never colorize output;", "auto|always|never"))
336 fn usage(binary: &str) {
337 let message = format!("Usage: {} [OPTIONS] [FILTER]", binary);
340 The FILTER string is tested against the name of all tests, and only those
341 tests whose names contain the filter are run.
343 By default, all tests are run in parallel. This can be altered with the
344 RUST_TEST_THREADS environment variable when running tests (set it to 1).
346 All tests have their standard output and standard error captured by default.
347 This can be overridden with the --nocapture flag or the RUST_TEST_NOCAPTURE=1
348 environment variable. Logging is not captured by default.
352 #[test] - Indicates a function is a test to be run. This function
354 #[bench] - Indicates a function is a benchmark to be run. This
355 function takes one argument (test::Bencher).
356 #[should_panic] - This function (also labeled with #[test]) will only pass if
357 the code causes a panic (an assertion failure or panic!)
358 A message may be provided, which the failure string must
359 contain: #[should_panic(expected = "foo")].
360 #[ignore] - When applied to a function which is already attributed as a
361 test, then the test runner will ignore these tests during
362 normal test runs. Running with --ignored will run these
364 usage = getopts::usage(&message, &optgroups()));
367 // Parses command line arguments into test options
368 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
369 let args_ = &args[1..];
370 let matches = match getopts::getopts(args_, &optgroups()) {
372 Err(f) => return Some(Err(f.to_string())),
375 if matches.opt_present("h") {
380 let filter = if !matches.free.is_empty() {
381 Some(matches.free[0].clone())
386 let run_ignored = matches.opt_present("ignored");
388 let logfile = matches.opt_str("logfile");
389 let logfile = logfile.map(|s| PathBuf::from(&s));
391 let bench_benchmarks = matches.opt_present("bench");
392 let run_tests = !bench_benchmarks || matches.opt_present("test");
394 let mut nocapture = matches.opt_present("nocapture");
396 nocapture = env::var("RUST_TEST_NOCAPTURE").is_ok();
399 let color = match matches.opt_str("color").as_ref().map(|s| &**s) {
400 Some("auto") | None => AutoColor,
401 Some("always") => AlwaysColor,
402 Some("never") => NeverColor,
405 return Some(Err(format!("argument for --color must be auto, always, or never (was \
411 let test_opts = TestOpts {
413 run_ignored: run_ignored,
414 run_tests: run_tests,
415 bench_benchmarks: bench_benchmarks,
417 nocapture: nocapture,
424 #[derive(Clone, PartialEq)]
425 pub struct BenchSamples {
426 ns_iter_summ: stats::Summary,
430 #[derive(Clone, PartialEq)]
431 pub enum TestResult {
435 TrMetrics(MetricMap),
436 TrBench(BenchSamples),
439 unsafe impl Send for TestResult {}
441 enum OutputLocation<T> {
442 Pretty(Box<term::StdoutTerminal>),
446 struct ConsoleTestState<T> {
447 log_out: Option<File>,
448 out: OutputLocation<T>,
456 failures: Vec<(TestDesc, Vec<u8>)>,
457 max_name_len: usize, // number of columns to fill when aligning names
460 impl<T: Write> ConsoleTestState<T> {
461 pub fn new(opts: &TestOpts, _: Option<T>) -> io::Result<ConsoleTestState<io::Stdout>> {
462 let log_out = match opts.logfile {
463 Some(ref path) => Some(try!(File::create(path))),
466 let out = match term::stdout() {
467 None => Raw(io::stdout()),
468 Some(t) => Pretty(t),
471 Ok(ConsoleTestState {
474 use_color: use_color(opts),
480 metrics: MetricMap::new(),
481 failures: Vec::new(),
486 pub fn write_ok(&mut self) -> io::Result<()> {
487 self.write_pretty("ok", term::color::GREEN)
490 pub fn write_failed(&mut self) -> io::Result<()> {
491 self.write_pretty("FAILED", term::color::RED)
494 pub fn write_ignored(&mut self) -> io::Result<()> {
495 self.write_pretty("ignored", term::color::YELLOW)
498 pub fn write_metric(&mut self) -> io::Result<()> {
499 self.write_pretty("metric", term::color::CYAN)
502 pub fn write_bench(&mut self) -> io::Result<()> {
503 self.write_pretty("bench", term::color::CYAN)
506 pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> {
508 Pretty(ref mut term) => {
510 try!(term.fg(color));
512 try!(term.write_all(word.as_bytes()));
518 Raw(ref mut stdout) => {
519 try!(stdout.write_all(word.as_bytes()));
525 pub fn write_plain(&mut self, s: &str) -> io::Result<()> {
527 Pretty(ref mut term) => {
528 try!(term.write_all(s.as_bytes()));
531 Raw(ref mut stdout) => {
532 try!(stdout.write_all(s.as_bytes()));
538 pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
540 let noun = if len != 1 {
545 self.write_plain(&format!("\nrunning {} {}\n", len, noun))
548 pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
549 let name = test.padded_name(self.max_name_len, align);
550 self.write_plain(&format!("test {} ... ", name))
553 pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
555 TrOk => self.write_ok(),
556 TrFailed => self.write_failed(),
557 TrIgnored => self.write_ignored(),
558 TrMetrics(ref mm) => {
559 try!(self.write_metric());
560 self.write_plain(&format!(": {}", mm.fmt_metrics()))
563 try!(self.write_bench());
565 try!(self.write_plain(&format!(": {}", fmt_bench_samples(bs))));
570 self.write_plain("\n")
573 pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
577 let s = format!("{} {}\n",
579 TrOk => "ok".to_owned(),
580 TrFailed => "failed".to_owned(),
581 TrIgnored => "ignored".to_owned(),
582 TrMetrics(ref mm) => mm.fmt_metrics(),
583 TrBench(ref bs) => fmt_bench_samples(bs),
586 o.write_all(s.as_bytes())
591 pub fn write_failures(&mut self) -> io::Result<()> {
592 try!(self.write_plain("\nfailures:\n"));
593 let mut failures = Vec::new();
594 let mut fail_out = String::new();
595 for &(ref f, ref stdout) in &self.failures {
596 failures.push(f.name.to_string());
597 if !stdout.is_empty() {
598 fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
599 let output = String::from_utf8_lossy(stdout);
600 fail_out.push_str(&output);
601 fail_out.push_str("\n");
604 if !fail_out.is_empty() {
605 try!(self.write_plain("\n"));
606 try!(self.write_plain(&fail_out));
609 try!(self.write_plain("\nfailures:\n"));
611 for name in &failures {
612 try!(self.write_plain(&format!(" {}\n", name)));
617 pub fn write_run_finish(&mut self) -> io::Result<bool> {
618 assert!(self.passed + self.failed + self.ignored + self.measured == self.total);
620 let success = self.failed == 0;
622 try!(self.write_failures());
625 try!(self.write_plain("\ntest result: "));
627 // There's no parallelism at this point so it's safe to use color
628 try!(self.write_ok());
630 try!(self.write_failed());
632 let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n",
637 try!(self.write_plain(&s));
642 // Format a number with thousands separators
643 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
645 let mut output = String::new();
646 let mut trailing = false;
647 for &pow in &[9, 6, 3, 0] {
648 let base = 10_usize.pow(pow);
649 if pow == 0 || trailing || n / base != 0 {
651 output.write_fmt(format_args!("{}", n / base)).unwrap();
653 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
666 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
668 let mut output = String::new();
670 let median = bs.ns_iter_summ.median as usize;
671 let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
673 output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
674 fmt_thousands_sep(median, ','),
675 fmt_thousands_sep(deviation, ',')))
678 output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
683 // A simple console test runner
684 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
686 fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
687 match (*event).clone() {
688 TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
689 TeWait(ref test, padding) => st.write_test_start(test, padding),
690 TeResult(test, result, stdout) => {
691 try!(st.write_log(&test, &result));
692 try!(st.write_result(&result));
694 TrOk => st.passed += 1,
695 TrIgnored => st.ignored += 1,
697 let tname = test.name;
698 let MetricMap(mm) = mm;
701 .insert_metric(&format!("{}.{}", tname, k), v.value, v.noise);
706 st.metrics.insert_metric(test.name.as_slice(),
707 bs.ns_iter_summ.median,
708 bs.ns_iter_summ.max - bs.ns_iter_summ.min);
713 st.failures.push((test, stdout));
721 let mut st = try!(ConsoleTestState::new(opts, None::<io::Stdout>));
722 fn len_if_padded(t: &TestDescAndFn) -> usize {
723 match t.testfn.padding() {
725 PadOnRight => t.desc.name.as_slice().len(),
728 match tests.iter().max_by_key(|t| len_if_padded(*t)) {
730 let n = t.desc.name.as_slice();
731 st.max_name_len = n.len();
735 try!(run_tests(opts, tests, |x| callback(&x, &mut st)));
736 return st.write_run_finish();
740 fn should_sort_failures_before_printing_them() {
741 let test_a = TestDesc {
742 name: StaticTestName("a"),
744 should_panic: ShouldPanic::No,
747 let test_b = TestDesc {
748 name: StaticTestName("b"),
750 should_panic: ShouldPanic::No,
753 let mut st = ConsoleTestState {
755 out: Raw(Vec::new()),
763 metrics: MetricMap::new(),
764 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
767 st.write_failures().unwrap();
768 let s = match st.out {
769 Raw(ref m) => String::from_utf8_lossy(&m[..]),
770 Pretty(_) => unreachable!(),
773 let apos = s.find("a").unwrap();
774 let bpos = s.find("b").unwrap();
775 assert!(apos < bpos);
778 fn use_color(opts: &TestOpts) -> bool {
780 AutoColor => !opts.nocapture && stdout_isatty(),
787 fn stdout_isatty() -> bool {
788 unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
791 fn stdout_isatty() -> bool {
794 type HANDLE = *mut u8;
795 type LPDWORD = *mut u32;
796 const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD;
798 fn GetStdHandle(which: DWORD) -> HANDLE;
799 fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL;
802 let handle = GetStdHandle(STD_OUTPUT_HANDLE);
804 GetConsoleMode(handle, &mut out) != 0
810 TeFiltered(Vec<TestDesc>),
811 TeWait(TestDesc, NamePadding),
812 TeResult(TestDesc, TestResult, Vec<u8>),
815 pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
818 fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
819 where F: FnMut(TestEvent) -> io::Result<()>
821 let mut filtered_tests = filter_tests(opts, tests);
822 if !opts.bench_benchmarks {
823 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
826 let filtered_descs = filtered_tests.iter()
827 .map(|t| t.desc.clone())
830 try!(callback(TeFiltered(filtered_descs)));
832 let (filtered_tests, filtered_benchs_and_metrics): (Vec<_>, _) =
833 filtered_tests.into_iter().partition(|e| {
835 StaticTestFn(_) | DynTestFn(_) => true,
840 // It's tempting to just spawn all the tests at once, but since we have
841 // many tests that run in other processes we would be making a big mess.
842 let concurrency = get_concurrency();
844 let mut remaining = filtered_tests;
848 let (tx, rx) = channel::<MonitorMsg>();
850 while pending > 0 || !remaining.is_empty() {
851 while pending < concurrency && !remaining.is_empty() {
852 let test = remaining.pop().unwrap();
853 if concurrency == 1 {
854 // We are doing one test at a time so we can print the name
855 // of the test before we run it. Useful for debugging tests
856 // that hang forever.
857 try!(callback(TeWait(test.desc.clone(), test.testfn.padding())));
859 run_test(opts, !opts.run_tests, test, tx.clone());
863 let (desc, result, stdout) = rx.recv().unwrap();
864 if concurrency != 1 {
865 try!(callback(TeWait(desc.clone(), PadNone)));
867 try!(callback(TeResult(desc, result, stdout)));
871 if opts.bench_benchmarks {
872 // All benchmarks run at the end, in serial.
873 // (this includes metric fns)
874 for b in filtered_benchs_and_metrics {
875 try!(callback(TeWait(b.desc.clone(), b.testfn.padding())));
876 run_test(opts, false, b, tx.clone());
877 let (test, result, stdout) = rx.recv().unwrap();
878 try!(callback(TeResult(test, result, stdout)));
885 fn get_concurrency() -> usize {
886 return match env::var("RUST_TEST_THREADS") {
888 let opt_n: Option<usize> = s.parse().ok();
890 Some(n) if n > 0 => n,
892 panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.",
897 Err(..) => num_cpus(),
902 fn num_cpus() -> usize {
905 wProcessorArchitecture: u16,
908 lpMinimumApplicationAddress: *mut u8,
909 lpMaximumApplicationAddress: *mut u8,
910 dwActiveProcessorMask: *mut u8,
911 dwNumberOfProcessors: u32,
912 dwProcessorType: u32,
913 dwAllocationGranularity: u32,
914 wProcessorLevel: u16,
915 wProcessorRevision: u16,
918 fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32;
921 let mut sysinfo = std::mem::zeroed();
922 GetSystemInfo(&mut sysinfo);
923 sysinfo.dwNumberOfProcessors as usize
927 #[cfg(any(target_os = "linux",
930 target_os = "android"))]
931 fn num_cpus() -> usize {
932 unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
935 #[cfg(any(target_os = "freebsd",
936 target_os = "dragonfly",
937 target_os = "bitrig",
938 target_os = "netbsd"))]
939 fn num_cpus() -> usize {
940 let mut cpus: libc::c_uint = 0;
941 let mut cpus_size = std::mem::size_of_val(&cpus);
942 let mut mib = [libc::CTL_HW, libc::HW_AVAILCPU, 0, 0];
945 libc::sysctl(mib.as_mut_ptr(),
947 &mut cpus as *mut _ as *mut _,
948 &mut cpus_size as *mut _ as *mut _,
953 mib[1] = libc::HW_NCPU;
955 libc::sysctl(mib.as_mut_ptr(),
957 &mut cpus as *mut _ as *mut _,
958 &mut cpus_size as *mut _ as *mut _,
969 #[cfg(target_os = "openbsd")]
970 fn num_cpus() -> usize {
971 let mut cpus: libc::c_uint = 0;
972 let mut cpus_size = std::mem::size_of_val(&cpus);
973 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
976 libc::sysctl(mib.as_mut_ptr(),
978 &mut cpus as *mut _ as *mut _,
979 &mut cpus_size as *mut _ as *mut _,
990 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
991 let mut filtered = tests;
993 // Remove tests that don't match the test filter
994 filtered = match opts.filter {
996 Some(ref filter) => {
998 .filter(|test| test.desc.name.as_slice().contains(&filter[..]))
1003 // Maybe pull out the ignored test and unignore them
1004 filtered = if !opts.run_ignored {
1007 fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
1008 if test.desc.ignore {
1009 let TestDescAndFn {desc, testfn} = test;
1010 Some(TestDescAndFn {
1011 desc: TestDesc { ignore: false, ..desc },
1018 filtered.into_iter().filter_map(filter).collect()
1021 // Sort the tests alphabetically
1022 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
1027 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1028 // convert benchmarks to tests, if we're not benchmarking them
1031 let testfn = match x.testfn {
1032 DynBenchFn(bench) => {
1033 DynTestFn(Box::new(move || bench::run_once(|b| bench.run(b))))
1035 StaticBenchFn(benchfn) => {
1036 DynTestFn(Box::new(move || bench::run_once(|b| benchfn(b))))
1048 pub fn run_test(opts: &TestOpts,
1050 test: TestDescAndFn,
1051 monitor_ch: Sender<MonitorMsg>) {
1053 let TestDescAndFn {desc, testfn} = test;
1055 if force_ignore || desc.ignore {
1056 monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap();
1060 fn run_test_inner(desc: TestDesc,
1061 monitor_ch: Sender<MonitorMsg>,
1063 testfn: Box<FnBox() + Send>) {
1064 struct Sink(Arc<Mutex<Vec<u8>>>);
1065 impl Write for Sink {
1066 fn write(&mut self, data: &[u8]) -> io::Result<usize> {
1067 Write::write(&mut *self.0.lock().unwrap(), data)
1069 fn flush(&mut self) -> io::Result<()> {
1074 thread::spawn(move || {
1075 let data = Arc::new(Mutex::new(Vec::new()));
1076 let data2 = data.clone();
1077 let cfg = thread::Builder::new().name(match desc.name {
1078 DynTestName(ref name) => name.clone(),
1079 StaticTestName(name) => name.to_owned(),
1082 let result_guard = cfg.spawn(move || {
1084 io::set_print(box Sink(data2.clone()));
1085 io::set_panic(box Sink(data2));
1090 let test_result = calc_result(&desc, result_guard.join());
1091 let stdout = data.lock().unwrap().to_vec();
1092 monitor_ch.send((desc.clone(), test_result, stdout)).unwrap();
1097 DynBenchFn(bencher) => {
1098 let bs = ::bench::benchmark(|harness| bencher.run(harness));
1099 monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
1102 StaticBenchFn(benchfn) => {
1103 let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness));
1104 monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap();
1108 let mut mm = MetricMap::new();
1109 f.call_box((&mut mm,));
1110 monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap();
1113 StaticMetricFn(f) => {
1114 let mut mm = MetricMap::new();
1116 monitor_ch.send((desc, TrMetrics(mm), Vec::new())).unwrap();
1119 DynTestFn(f) => run_test_inner(desc, monitor_ch, opts.nocapture, f),
1120 StaticTestFn(f) => run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(f)),
1124 fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> TestResult {
1125 match (&desc.should_panic, task_result) {
1126 (&ShouldPanic::No, Ok(())) |
1127 (&ShouldPanic::Yes, Err(_)) => TrOk,
1128 (&ShouldPanic::YesWithMessage(msg), Err(ref err))
1129 if err.downcast_ref::<String>()
1131 .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
1132 .map(|e| e.contains(msg))
1133 .unwrap_or(false) => TrOk,
1139 pub fn new() -> MetricMap {
1140 MetricMap(BTreeMap::new())
1143 /// Insert a named `value` (+/- `noise`) metric into the map. The value
1144 /// must be non-negative. The `noise` indicates the uncertainty of the
1145 /// metric, which doubles as the "noise range" of acceptable
1146 /// pairwise-regressions on this named value, when comparing from one
1147 /// metric to the next using `compare_to_old`.
1149 /// If `noise` is positive, then it means this metric is of a value
1150 /// you want to see grow smaller, so a change larger than `noise` in the
1151 /// positive direction represents a regression.
1153 /// If `noise` is negative, then it means this metric is of a value
1154 /// you want to see grow larger, so a change larger than `noise` in the
1155 /// negative direction represents a regression.
1156 pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) {
1161 let MetricMap(ref mut map) = *self;
1162 map.insert(name.to_owned(), m);
1165 pub fn fmt_metrics(&self) -> String {
1166 let MetricMap(ref mm) = *self;
1167 let v: Vec<String> = mm.iter()
1168 .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
1177 /// A function that is opaque to the optimizer, to allow benchmarks to
1178 /// pretend to use outputs to assist in avoiding dead-code
1181 /// This function is a no-op, and does not even read from `dummy`.
1182 #[cfg(not(all(target_os = "nacl", target_arch = "le32")))]
1183 pub fn black_box<T>(dummy: T) -> T {
1184 // we need to "use" the argument in some way LLVM can't
1186 unsafe { asm!("" : : "r"(&dummy)) }
1189 #[cfg(all(target_os = "nacl", target_arch = "le32"))]
1191 pub fn black_box<T>(dummy: T) -> T {
1197 /// Callback for benchmark functions to run in their body.
1198 pub fn iter<T, F>(&mut self, mut inner: F)
1199 where F: FnMut() -> T
1201 let start = Instant::now();
1202 let k = self.iterations;
1206 self.dur = start.elapsed();
1209 pub fn ns_elapsed(&mut self) -> u64 {
1210 self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64)
1213 pub fn ns_per_iter(&mut self) -> u64 {
1214 if self.iterations == 0 {
1217 self.ns_elapsed() / cmp::max(self.iterations, 1)
1221 pub fn bench_n<F>(&mut self, n: u64, f: F)
1222 where F: FnOnce(&mut Bencher)
1224 self.iterations = n;
1228 // This is a more statistics-driven benchmark algorithm
1229 pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary
1230 where F: FnMut(&mut Bencher)
1232 // Initial bench run to get ballpark figure.
1234 self.bench_n(n, |x| f(x));
1236 // Try to estimate iter count for 1ms falling back to 1m
1237 // iterations if first run took < 1ns.
1238 if self.ns_per_iter() == 0 {
1241 n = 1_000_000 / cmp::max(self.ns_per_iter(), 1);
1243 // if the first run took more than 1ms we don't want to just
1244 // be left doing 0 iterations on every loop. The unfortunate
1245 // side effect of not being able to do as many runs is
1246 // automatically handled by the statistical analysis below
1247 // (i.e. larger error bars).
1252 let mut total_run = Duration::new(0, 0);
1253 let samples: &mut [f64] = &mut [0.0_f64; 50];
1255 let loop_start = Instant::now();
1257 for p in &mut *samples {
1258 self.bench_n(n, |x| f(x));
1259 *p = self.ns_per_iter() as f64;
1262 stats::winsorize(samples, 5.0);
1263 let summ = stats::Summary::new(samples);
1265 for p in &mut *samples {
1266 self.bench_n(5 * n, |x| f(x));
1267 *p = self.ns_per_iter() as f64;
1270 stats::winsorize(samples, 5.0);
1271 let summ5 = stats::Summary::new(samples);
1272 let loop_run = loop_start.elapsed();
1274 // If we've run for 100ms and seem to have converged to a
1276 if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
1277 summ.median - summ5.median < summ5.median_abs_dev {
1281 total_run = total_run + loop_run;
1282 // Longest we ever run for is 3s.
1283 if total_run > Duration::from_secs(3) {
1287 // If we overflow here just return the results so far. We check a
1288 // multiplier of 10 because we're about to multiply by 2 and the
1289 // next iteration of the loop will also multiply by 5 (to calculate
1290 // the summ5 result)
1291 n = match n.checked_mul(10) {
1293 None => return summ5,
1301 use std::time::Duration;
1302 use super::{Bencher, BenchSamples};
1304 pub fn benchmark<F>(f: F) -> BenchSamples
1305 where F: FnMut(&mut Bencher)
1307 let mut bs = Bencher {
1309 dur: Duration::new(0, 0),
1313 let ns_iter_summ = bs.auto_bench(f);
1315 let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
1316 let mb_s = bs.bytes * 1000 / ns_iter;
1319 ns_iter_summ: ns_iter_summ,
1320 mb_s: mb_s as usize,
1324 pub fn run_once<F>(f: F)
1325 where F: FnOnce(&mut Bencher)
1327 let mut bs = Bencher {
1329 dur: Duration::new(0, 0),
1338 use test::{TrFailed, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc, TestDescAndFn,
1339 TestOpts, run_test, MetricMap, StaticTestName, DynTestName, DynTestFn, ShouldPanic};
1340 use std::sync::mpsc::channel;
1343 pub fn do_not_run_ignored_tests() {
1347 let desc = TestDescAndFn {
1349 name: StaticTestName("whatever"),
1351 should_panic: ShouldPanic::No,
1353 testfn: DynTestFn(Box::new(move || f())),
1355 let (tx, rx) = channel();
1356 run_test(&TestOpts::new(), false, desc, tx);
1357 let (_, res, _) = rx.recv().unwrap();
1358 assert!(res != TrOk);
1362 pub fn ignored_tests_result_in_ignored() {
1364 let desc = TestDescAndFn {
1366 name: StaticTestName("whatever"),
1368 should_panic: ShouldPanic::No,
1370 testfn: DynTestFn(Box::new(move || f())),
1372 let (tx, rx) = channel();
1373 run_test(&TestOpts::new(), false, desc, tx);
1374 let (_, res, _) = rx.recv().unwrap();
1375 assert!(res == TrIgnored);
1379 fn test_should_panic() {
1383 let desc = TestDescAndFn {
1385 name: StaticTestName("whatever"),
1387 should_panic: ShouldPanic::Yes,
1389 testfn: DynTestFn(Box::new(move || f())),
1391 let (tx, rx) = channel();
1392 run_test(&TestOpts::new(), false, desc, tx);
1393 let (_, res, _) = rx.recv().unwrap();
1394 assert!(res == TrOk);
1398 fn test_should_panic_good_message() {
1400 panic!("an error message");
1402 let desc = TestDescAndFn {
1404 name: StaticTestName("whatever"),
1406 should_panic: ShouldPanic::YesWithMessage("error message"),
1408 testfn: DynTestFn(Box::new(move || f())),
1410 let (tx, rx) = channel();
1411 run_test(&TestOpts::new(), false, desc, tx);
1412 let (_, res, _) = rx.recv().unwrap();
1413 assert!(res == TrOk);
1417 fn test_should_panic_bad_message() {
1419 panic!("an error message");
1421 let desc = TestDescAndFn {
1423 name: StaticTestName("whatever"),
1425 should_panic: ShouldPanic::YesWithMessage("foobar"),
1427 testfn: DynTestFn(Box::new(move || f())),
1429 let (tx, rx) = channel();
1430 run_test(&TestOpts::new(), false, desc, tx);
1431 let (_, res, _) = rx.recv().unwrap();
1432 assert!(res == TrFailed);
1436 fn test_should_panic_but_succeeds() {
1438 let desc = TestDescAndFn {
1440 name: StaticTestName("whatever"),
1442 should_panic: ShouldPanic::Yes,
1444 testfn: DynTestFn(Box::new(move || f())),
1446 let (tx, rx) = channel();
1447 run_test(&TestOpts::new(), false, desc, tx);
1448 let (_, res, _) = rx.recv().unwrap();
1449 assert!(res == TrFailed);
1453 fn parse_ignored_flag() {
1454 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
1455 let opts = match parse_opts(&args) {
1457 _ => panic!("Malformed arg in parse_ignored_flag"),
1459 assert!((opts.run_ignored));
1463 pub fn filter_for_ignored_option() {
1464 // When we run ignored tests the test filter should filter out all the
1465 // unignored tests and flip the ignore flag on the rest to false
1467 let mut opts = TestOpts::new();
1468 opts.run_tests = true;
1469 opts.run_ignored = true;
1471 let tests = vec![TestDescAndFn {
1473 name: StaticTestName("1"),
1475 should_panic: ShouldPanic::No,
1477 testfn: DynTestFn(Box::new(move || {})),
1481 name: StaticTestName("2"),
1483 should_panic: ShouldPanic::No,
1485 testfn: DynTestFn(Box::new(move || {})),
1487 let filtered = filter_tests(&opts, tests);
1489 assert_eq!(filtered.len(), 1);
1490 assert_eq!(filtered[0].desc.name.to_string(), "1");
1491 assert!(filtered[0].desc.ignore == false);
1495 pub fn sort_tests() {
1496 let mut opts = TestOpts::new();
1497 opts.run_tests = true;
1499 let names = vec!["sha1::test".to_string(),
1500 "isize::test_to_str".to_string(),
1501 "isize::test_pow".to_string(),
1502 "test::do_not_run_ignored_tests".to_string(),
1503 "test::ignored_tests_result_in_ignored".to_string(),
1504 "test::first_free_arg_should_be_a_filter".to_string(),
1505 "test::parse_ignored_flag".to_string(),
1506 "test::filter_for_ignored_option".to_string(),
1507 "test::sort_tests".to_string()];
1510 let mut tests = Vec::new();
1511 for name in &names {
1512 let test = TestDescAndFn {
1514 name: DynTestName((*name).clone()),
1516 should_panic: ShouldPanic::No,
1518 testfn: DynTestFn(Box::new(testfn)),
1524 let filtered = filter_tests(&opts, tests);
1526 let expected = vec!["isize::test_pow".to_string(),
1527 "isize::test_to_str".to_string(),
1528 "sha1::test".to_string(),
1529 "test::do_not_run_ignored_tests".to_string(),
1530 "test::filter_for_ignored_option".to_string(),
1531 "test::first_free_arg_should_be_a_filter".to_string(),
1532 "test::ignored_tests_result_in_ignored".to_string(),
1533 "test::parse_ignored_flag".to_string(),
1534 "test::sort_tests".to_string()];
1536 for (a, b) in expected.iter().zip(filtered) {
1537 assert!(*a == b.desc.name.to_string());
1542 pub fn test_metricmap_compare() {
1543 let mut m1 = MetricMap::new();
1544 let mut m2 = MetricMap::new();
1545 m1.insert_metric("in-both-noise", 1000.0, 200.0);
1546 m2.insert_metric("in-both-noise", 1100.0, 200.0);
1548 m1.insert_metric("in-first-noise", 1000.0, 2.0);
1549 m2.insert_metric("in-second-noise", 1000.0, 2.0);
1551 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
1552 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
1554 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
1555 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
1557 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
1558 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
1560 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
1561 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);