1 // Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Support code for rustc's built in unit-test and micro-benchmarking
14 //! Almost all user code will only be interested in `Bencher` and
15 //! `black_box`. All other interactions (such as writing tests and
16 //! benchmarks themselves) should be done via the `#[test]` and
17 //! `#[bench]` attributes.
19 //! See the [Testing Chapter](../book/first-edition/testing.html) of the book for more details.
21 // Currently, not much of this is meant for users. It is intended to
22 // support the simplest interface possible for representing and
23 // running tests while providing a base that other test frameworks may
26 // NB: this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
27 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
28 // cargo) to detect this crate.
29 #![crate_name = "test"]
30 #![unstable(feature = "test", issue = "27812")]
31 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
32 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
33 html_root_url = "https://doc.rust-lang.org/nightly/",
34 test(attr(deny(warnings))))]
39 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
40 #![feature(set_stdio)]
41 #![feature(panic_unwind)]
42 #![feature(staged_api)]
46 #[cfg(any(unix, target_os = "cloudabi"))]
48 extern crate panic_unwind;
50 pub use self::TestFn::*;
51 pub use self::ColorConfig::*;
52 pub use self::TestResult::*;
53 pub use self::TestName::*;
54 use self::TestEvent::*;
55 use self::NamePadding::*;
56 use self::OutputLocation::*;
58 use std::panic::{catch_unwind, AssertUnwindSafe};
60 use std::boxed::FnBox;
62 use std::collections::BTreeMap;
66 use std::io::prelude::*;
68 use std::iter::repeat;
69 use std::path::PathBuf;
70 use std::sync::mpsc::{channel, Sender};
71 use std::sync::{Arc, Mutex};
73 use std::time::{Instant, Duration};
76 const TEST_WARN_TIMEOUT_S: u64 = 60;
77 const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode
79 // to be used by rustc to compile tests in libtest
81 pub use {Bencher, TestName, TestResult, TestDesc, TestDescAndFn, TestOpts, TrFailed,
82 TrFailedMsg, TrIgnored, TrOk, Metric, MetricMap, StaticTestFn, StaticTestName,
83 DynTestName, DynTestFn, run_test, test_main, test_main_static, filter_tests,
84 parse_opts, StaticBenchFn, ShouldPanic, Options};
90 use formatters::{OutputFormatter, PrettyFormatter, TerseFormatter, JsonFormatter};
92 // The name of a test. By convention this follows the rules for rust
93 // paths; i.e. it should be a series of identifiers separated by double
94 // colons. This way if some test runner wants to arrange the tests
95 // hierarchically it may.
97 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
99 StaticTestName(&'static str),
101 AlignedTestName(Cow<'static, str>, NamePadding),
104 fn as_slice(&self) -> &str {
106 StaticTestName(s) => s,
107 DynTestName(ref s) => s,
108 AlignedTestName(ref s, _) => &*s,
112 fn padding(&self) -> NamePadding {
114 &AlignedTestName(_, p) => p,
119 fn with_padding(&self, padding: NamePadding) -> TestName {
120 let name = match self {
121 &TestName::StaticTestName(name) => Cow::Borrowed(name),
122 &TestName::DynTestName(ref name) => Cow::Owned(name.clone()),
123 &TestName::AlignedTestName(ref name, _) => name.clone(),
126 TestName::AlignedTestName(name, padding)
129 impl fmt::Display for TestName {
130 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
131 fmt::Display::fmt(self.as_slice(), f)
135 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
136 pub enum NamePadding {
142 fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
143 let mut name = String::from(self.name.as_slice());
144 let fill = column_count.saturating_sub(name.len());
145 let pad = repeat(" ").take(fill).collect::<String>();
156 /// Represents a benchmark function.
157 pub trait TDynBenchFn: Send {
158 fn run(&self, harness: &mut Bencher);
161 // A function that runs a test. If the function returns successfully,
162 // the test succeeds; if the function panics then the test fails. We
163 // may need to come up with a more clever definition of test in order
164 // to support isolation of tests into threads.
167 StaticBenchFn(fn(&mut Bencher)),
168 DynTestFn(Box<FnBox() + Send>),
169 DynBenchFn(Box<TDynBenchFn + 'static>),
173 fn padding(&self) -> NamePadding {
175 StaticTestFn(..) => PadNone,
176 StaticBenchFn(..) => PadOnRight,
177 DynTestFn(..) => PadNone,
178 DynBenchFn(..) => PadOnRight,
183 impl fmt::Debug for TestFn {
184 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
185 f.write_str(match *self {
186 StaticTestFn(..) => "StaticTestFn(..)",
187 StaticBenchFn(..) => "StaticBenchFn(..)",
188 DynTestFn(..) => "DynTestFn(..)",
189 DynBenchFn(..) => "DynBenchFn(..)",
194 /// Manager of the benchmarking runs.
196 /// This is fed into functions marked with `#[bench]` to allow for
197 /// set-up & tear-down before running a piece of code repeatedly via a
202 summary: Option<stats::Summary>,
206 #[derive(Clone, PartialEq, Eq)]
212 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
213 pub enum ShouldPanic {
216 YesWithMessage(&'static str),
219 // The definition of a single test. A test runner will run a list of
221 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
222 pub struct TestDesc {
225 pub should_panic: ShouldPanic,
226 pub allow_fail: bool,
230 pub struct TestDescAndFn {
235 #[derive(Clone, PartialEq, Debug, Copy)]
242 pub fn new(value: f64, noise: f64) -> Metric {
243 Metric { value, noise }
247 /// In case we want to add other options as well, just add them in this struct.
248 #[derive(Copy, Clone, Debug)]
250 display_output: bool,
254 pub fn new() -> Options {
255 Options { display_output: false }
258 pub fn display_output(mut self, display_output: bool) -> Options {
259 self.display_output = display_output;
264 // The default console test runner. It accepts the command line
265 // arguments and a vector of test_descs.
266 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) {
267 let mut opts = match parse_opts(args) {
269 Some(Err(msg)) => panic!("{:?}", msg),
272 opts.options = options;
274 if let Err(e) = list_tests_console(&opts, tests) {
275 panic!("io error when listing tests: {:?}", e);
278 match run_tests_console(&opts, tests) {
280 Ok(false) => std::process::exit(101),
281 Err(e) => panic!("io error when running tests: {:?}", e),
286 // A variant optimized for invocation with a static test vector.
287 // This will panic (intentionally) when fed any dynamic tests, because
288 // it is copying the static values out into a dynamic vector and cannot
289 // copy dynamic values. It is doing this because from this point on
290 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
291 // semantics into parallel test runners, which in turn requires a Vec<>
292 // rather than a &[].
293 pub fn test_main_static(tests: &[TestDescAndFn]) {
294 let args = env::args().collect::<Vec<_>>();
295 let owned_tests = tests
297 .map(|t| match t.testfn {
300 testfn: StaticTestFn(f),
301 desc: t.desc.clone(),
304 StaticBenchFn(f) => {
306 testfn: StaticBenchFn(f),
307 desc: t.desc.clone(),
310 _ => panic!("non-static tests passed to test::test_main_static"),
313 test_main(&args, owned_tests, Options::new())
316 #[derive(Copy, Clone, Debug)]
317 pub enum ColorConfig {
323 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
324 pub enum OutputFormat {
331 pub struct TestOpts {
333 pub filter: Option<String>,
334 pub filter_exact: bool,
335 pub run_ignored: bool,
337 pub bench_benchmarks: bool,
338 pub logfile: Option<PathBuf>,
340 pub color: ColorConfig,
341 pub format: OutputFormat,
342 pub test_threads: Option<usize>,
343 pub skip: Vec<String>,
344 pub options: Options,
349 fn new() -> TestOpts {
356 bench_benchmarks: false,
360 format: OutputFormat::Pretty,
363 options: Options::new(),
368 /// Result of parsing the options.
369 pub type OptRes = Result<TestOpts, String>;
371 fn optgroups() -> getopts::Options {
372 let mut opts = getopts::Options::new();
373 opts.optflag("", "ignored", "Run ignored tests")
374 .optflag("", "test", "Run tests and not benchmarks")
375 .optflag("", "bench", "Run benchmarks instead of tests")
376 .optflag("", "list", "List all tests and benchmarks")
377 .optflag("h", "help", "Display this message (longer with --help)")
381 "Write logs to the specified file instead \
388 "don't capture stdout/stderr of each \
389 task, allow printing directly",
394 "Number of threads used for running tests \
401 "Skip tests whose names contain FILTER (this flag can \
402 be used multiple times)",
408 "Display one character per test instead of one line. \
409 Alias to --format=terse",
414 "Exactly match filters rather than by substring",
419 "Configure coloring of output:
420 auto = colorize if stdout is a tty and tests are run on serially (default);
421 always = always colorize output;
422 never = never colorize output;",
428 "Configure formatting of output:
429 pretty = Print verbose output;
430 terse = Display one character per test;
431 json = Output a json document",
437 "Enable nightly-only flags:
438 unstable-options = Allow use of experimental features",
444 fn usage(binary: &str, options: &getopts::Options) {
445 let message = format!("Usage: {} [OPTIONS] [FILTER]", binary);
449 The FILTER string is tested against the name of all tests, and only those
450 tests whose names contain the filter are run.
452 By default, all tests are run in parallel. This can be altered with the
453 --test-threads flag or the RUST_TEST_THREADS environment variable when running
456 All tests have their standard output and standard error captured by default.
457 This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE
458 environment variable to a value other than "0". Logging is not captured by default.
462 #[test] - Indicates a function is a test to be run. This function
464 #[bench] - Indicates a function is a benchmark to be run. This
465 function takes one argument (test::Bencher).
466 #[should_panic] - This function (also labeled with #[test]) will only pass if
467 the code causes a panic (an assertion failure or panic!)
468 A message may be provided, which the failure string must
469 contain: #[should_panic(expected = "foo")].
470 #[ignore] - When applied to a function which is already attributed as a
471 test, then the test runner will ignore these tests during
472 normal test runs. Running with --ignored will run these
474 usage = options.usage(&message)
478 // FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566
479 fn is_nightly() -> bool {
480 // Whether this is a feature-staged build, i.e. on the beta or stable channel
481 let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some();
482 // Whether we should enable unstable features for bootstrapping
483 let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
485 bootstrap || !disable_unstable_features
488 // Parses command line arguments into test options
489 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
490 let mut allow_unstable = false;
491 let opts = optgroups();
492 let args = args.get(1..).unwrap_or(args);
493 let matches = match opts.parse(args) {
495 Err(f) => return Some(Err(f.to_string())),
498 if let Some(opt) = matches.opt_str("Z") {
501 "the option `Z` is only accepted on the nightly compiler"
507 "unstable-options" => {
508 allow_unstable = true;
511 return Some(Err("Unrecognized option to `Z`".into()));
516 if matches.opt_present("h") {
517 usage(&args[0], &opts);
521 let filter = if !matches.free.is_empty() {
522 Some(matches.free[0].clone())
527 let run_ignored = matches.opt_present("ignored");
528 let quiet = matches.opt_present("quiet");
529 let exact = matches.opt_present("exact");
530 let list = matches.opt_present("list");
532 let logfile = matches.opt_str("logfile");
533 let logfile = logfile.map(|s| PathBuf::from(&s));
535 let bench_benchmarks = matches.opt_present("bench");
536 let run_tests = !bench_benchmarks || matches.opt_present("test");
538 let mut nocapture = matches.opt_present("nocapture");
540 nocapture = match env::var("RUST_TEST_NOCAPTURE") {
541 Ok(val) => &val != "0",
546 let test_threads = match matches.opt_str("test-threads") {
548 match n_str.parse::<usize>() {
549 Ok(0) => return Some(Err(format!("argument for --test-threads must not be 0"))),
552 return Some(Err(format!(
553 "argument for --test-threads must be a number > 0 \
563 let color = match matches.opt_str("color").as_ref().map(|s| &**s) {
564 Some("auto") | None => AutoColor,
565 Some("always") => AlwaysColor,
566 Some("never") => NeverColor,
569 return Some(Err(format!(
570 "argument for --color must be auto, always, or never (was \
577 let format = match matches.opt_str("format").as_ref().map(|s| &**s) {
578 None if quiet => OutputFormat::Terse,
579 Some("pretty") | None => OutputFormat::Pretty,
580 Some("terse") => OutputFormat::Terse,
584 "The \"json\" format is only accepted on the nightly compiler"
592 return Some(Err(format!(
593 "argument for --format must be pretty, terse, or json (was \
600 let test_opts = TestOpts {
612 skip: matches.opt_strs("skip"),
613 options: Options::new(),
619 #[derive(Clone, PartialEq)]
620 pub struct BenchSamples {
621 ns_iter_summ: stats::Summary,
625 #[derive(Clone, PartialEq)]
626 pub enum TestResult {
632 TrBench(BenchSamples),
635 unsafe impl Send for TestResult {}
637 enum OutputLocation<T> {
638 Pretty(Box<term::StdoutTerminal>),
642 impl<T: Write> Write for OutputLocation<T> {
643 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
645 Pretty(ref mut term) => term.write(buf),
646 Raw(ref mut stdout) => stdout.write(buf),
650 fn flush(&mut self) -> io::Result<()> {
652 Pretty(ref mut term) => term.flush(),
653 Raw(ref mut stdout) => stdout.flush(),
658 struct ConsoleTestState {
659 log_out: Option<File>,
668 failures: Vec<(TestDesc, Vec<u8>)>,
669 not_failures: Vec<(TestDesc, Vec<u8>)>,
673 impl ConsoleTestState {
674 pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState> {
675 let log_out = match opts.logfile {
676 Some(ref path) => Some(File::create(path)?),
680 Ok(ConsoleTestState {
689 metrics: MetricMap::new(),
690 failures: Vec::new(),
691 not_failures: Vec::new(),
692 options: opts.options,
696 pub fn write_log<S: AsRef<str>>(&mut self, msg: S) -> io::Result<()> {
697 let msg = msg.as_ref();
700 Some(ref mut o) => o.write_all(msg.as_bytes()),
704 pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
705 self.write_log(format!(
708 TrOk => "ok".to_owned(),
709 TrFailed => "failed".to_owned(),
710 TrFailedMsg(ref msg) => format!("failed: {}", msg),
711 TrIgnored => "ignored".to_owned(),
712 TrAllowedFail => "failed (allowed)".to_owned(),
713 TrBench(ref bs) => fmt_bench_samples(bs),
719 fn current_test_count(&self) -> usize {
720 self.passed + self.failed + self.ignored + self.measured + self.allowed_fail
724 // Format a number with thousands separators
725 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
727 let mut output = String::new();
728 let mut trailing = false;
729 for &pow in &[9, 6, 3, 0] {
730 let base = 10_usize.pow(pow);
731 if pow == 0 || trailing || n / base != 0 {
733 output.write_fmt(format_args!("{}", n / base)).unwrap();
735 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
748 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
750 let mut output = String::new();
752 let median = bs.ns_iter_summ.median as usize;
753 let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
756 .write_fmt(format_args!(
757 "{:>11} ns/iter (+/- {})",
758 fmt_thousands_sep(median, ','),
759 fmt_thousands_sep(deviation, ',')
764 .write_fmt(format_args!(" = {} MB/s", bs.mb_s))
770 // List the tests to console, and optionally to logfile. Filters are honored.
771 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
772 let mut output = match term::stdout() {
773 None => Raw(io::stdout()),
774 Some(t) => Pretty(t),
777 let quiet = opts.format == OutputFormat::Terse;
778 let mut st = ConsoleTestState::new(opts)?;
783 for test in filter_tests(&opts, tests) {
787 desc: TestDesc { name, .. },
791 let fntype = match testfn {
792 StaticTestFn(..) | DynTestFn(..) => {
803 writeln!(output, "{}: {}", name, fntype)?;
804 st.write_log(format!("{} {}\n", fntype, name))?;
807 fn plural(count: u32, s: &str) -> String {
809 1 => format!("{} {}", 1, s),
810 n => format!("{} {}s", n, s),
815 if ntest != 0 || nbench != 0 {
816 writeln!(output, "")?;
821 plural(ntest, "test"),
822 plural(nbench, "benchmark")
829 // A simple console test runner
830 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
833 st: &mut ConsoleTestState,
834 out: &mut OutputFormatter,
835 ) -> io::Result<()> {
837 match (*event).clone() {
838 TeFiltered(ref filtered_tests) => {
839 st.total = filtered_tests.len();
840 out.write_run_start(filtered_tests.len())
842 TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out),
843 TeWait(ref test) => out.write_test_start(test),
844 TeTimeout(ref test) => out.write_timeout(test),
845 TeResult(test, result, stdout) => {
846 st.write_log_result(&test, &result)?;
847 out.write_result(&test, &result, &*stdout)?;
851 st.not_failures.push((test, stdout));
853 TrIgnored => st.ignored += 1,
854 TrAllowedFail => st.allowed_fail += 1,
856 st.metrics.insert_metric(
857 test.name.as_slice(),
858 bs.ns_iter_summ.median,
859 bs.ns_iter_summ.max - bs.ns_iter_summ.min,
865 st.failures.push((test, stdout));
867 TrFailedMsg(msg) => {
869 let mut stdout = stdout;
870 stdout.extend_from_slice(format!("note: {}", msg).as_bytes());
871 st.failures.push((test, stdout));
879 let output = match term::stdout() {
880 None => Raw(io::stdout()),
881 Some(t) => Pretty(t),
884 let max_name_len = tests
886 .max_by_key(|t| len_if_padded(*t))
887 .map(|t| t.desc.name.as_slice().len())
890 let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1;
892 let mut out: Box<OutputFormatter> = match opts.format {
893 OutputFormat::Pretty => Box::new(PrettyFormatter::new(
899 OutputFormat::Terse => Box::new(TerseFormatter::new(
905 OutputFormat::Json => Box::new(JsonFormatter::new(output)),
907 let mut st = ConsoleTestState::new(opts)?;
908 fn len_if_padded(t: &TestDescAndFn) -> usize {
909 match t.testfn.padding() {
911 PadOnRight => t.desc.name.as_slice().len(),
915 run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?;
917 assert!(st.current_test_count() == st.total);
919 return out.write_run_finish(&st);
923 fn should_sort_failures_before_printing_them() {
924 let test_a = TestDesc {
925 name: StaticTestName("a"),
927 should_panic: ShouldPanic::No,
931 let test_b = TestDesc {
932 name: StaticTestName("b"),
934 should_panic: ShouldPanic::No,
938 let mut out = PrettyFormatter::new(Raw(Vec::new()), false, 10, false);
940 let st = ConsoleTestState {
949 metrics: MetricMap::new(),
950 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
951 options: Options::new(),
952 not_failures: Vec::new(),
955 out.write_failures(&st).unwrap();
956 let s = match out.output_location() {
957 &Raw(ref m) => String::from_utf8_lossy(&m[..]),
958 &Pretty(_) => unreachable!(),
961 let apos = s.find("a").unwrap();
962 let bpos = s.find("b").unwrap();
963 assert!(apos < bpos);
966 fn use_color(opts: &TestOpts) -> bool {
968 AutoColor => !opts.nocapture && stdout_isatty(),
974 #[cfg(any(target_os = "cloudabi",
976 all(target_arch = "wasm32", not(target_os = "emscripten"))))]
977 fn stdout_isatty() -> bool {
978 // FIXME: Implement isatty on Redox
982 fn stdout_isatty() -> bool {
983 unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
986 fn stdout_isatty() -> bool {
989 type HANDLE = *mut u8;
990 type LPDWORD = *mut u32;
991 const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD;
993 fn GetStdHandle(which: DWORD) -> HANDLE;
994 fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL;
997 let handle = GetStdHandle(STD_OUTPUT_HANDLE);
999 GetConsoleMode(handle, &mut out) != 0
1004 pub enum TestEvent {
1005 TeFiltered(Vec<TestDesc>),
1007 TeResult(TestDesc, TestResult, Vec<u8>),
1008 TeTimeout(TestDesc),
1009 TeFilteredOut(usize),
1012 pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
1014 struct Sink(Arc<Mutex<Vec<u8>>>);
1015 impl Write for Sink {
1016 fn write(&mut self, data: &[u8]) -> io::Result<usize> {
1017 Write::write(&mut *self.0.lock().unwrap(), data)
1019 fn flush(&mut self) -> io::Result<()> {
1024 pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
1026 F: FnMut(TestEvent) -> io::Result<()>,
1028 use std::collections::HashMap;
1029 use std::sync::mpsc::RecvTimeoutError;
1031 let tests_len = tests.len();
1033 let mut filtered_tests = filter_tests(opts, tests);
1034 if !opts.bench_benchmarks {
1035 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
1038 let filtered_tests = {
1039 let mut filtered_tests = filtered_tests;
1040 for test in filtered_tests.iter_mut() {
1041 test.desc.name = test.desc.name.with_padding(test.testfn.padding());
1047 let filtered_out = tests_len - filtered_tests.len();
1048 callback(TeFilteredOut(filtered_out))?;
1050 let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
1052 callback(TeFiltered(filtered_descs))?;
1054 let (filtered_tests, filtered_benchs): (Vec<_>, _) =
1055 filtered_tests.into_iter().partition(|e| match e.testfn {
1056 StaticTestFn(_) | DynTestFn(_) => true,
1060 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
1062 let mut remaining = filtered_tests;
1063 remaining.reverse();
1064 let mut pending = 0;
1066 let (tx, rx) = channel::<MonitorMsg>();
1068 let mut running_tests: HashMap<TestDesc, Instant> = HashMap::new();
1070 fn get_timed_out_tests(running_tests: &mut HashMap<TestDesc, Instant>) -> Vec<TestDesc> {
1071 let now = Instant::now();
1072 let timed_out = running_tests
1074 .filter_map(|(desc, timeout)| if &now >= timeout {
1080 for test in &timed_out {
1081 running_tests.remove(test);
1086 fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration> {
1087 running_tests.values().min().map(|next_timeout| {
1088 let now = Instant::now();
1089 if *next_timeout >= now {
1097 if concurrency == 1 {
1098 while !remaining.is_empty() {
1099 let test = remaining.pop().unwrap();
1100 callback(TeWait(test.desc.clone()))?;
1101 run_test(opts, !opts.run_tests, test, tx.clone());
1102 let (test, result, stdout) = rx.recv().unwrap();
1103 callback(TeResult(test, result, stdout))?;
1106 while pending > 0 || !remaining.is_empty() {
1107 while pending < concurrency && !remaining.is_empty() {
1108 let test = remaining.pop().unwrap();
1109 let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S);
1110 running_tests.insert(test.desc.clone(), timeout);
1111 callback(TeWait(test.desc.clone()))?; //here no pad
1112 run_test(opts, !opts.run_tests, test, tx.clone());
1118 if let Some(timeout) = calc_timeout(&running_tests) {
1119 res = rx.recv_timeout(timeout);
1120 for test in get_timed_out_tests(&mut running_tests) {
1121 callback(TeTimeout(test))?;
1123 if res != Err(RecvTimeoutError::Timeout) {
1127 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
1132 let (desc, result, stdout) = res.unwrap();
1133 running_tests.remove(&desc);
1135 callback(TeResult(desc, result, stdout))?;
1140 if opts.bench_benchmarks {
1141 // All benchmarks run at the end, in serial.
1142 for b in filtered_benchs {
1143 callback(TeWait(b.desc.clone()))?;
1144 run_test(opts, false, b, tx.clone());
1145 let (test, result, stdout) = rx.recv().unwrap();
1146 callback(TeResult(test, result, stdout))?;
1152 #[allow(deprecated)]
1153 fn get_concurrency() -> usize {
1154 return match env::var("RUST_TEST_THREADS") {
1156 let opt_n: Option<usize> = s.parse().ok();
1158 Some(n) if n > 0 => n,
1161 "RUST_TEST_THREADS is `{}`, should be a positive integer.",
1167 Err(..) => num_cpus(),
1172 fn num_cpus() -> usize {
1174 struct SYSTEM_INFO {
1175 wProcessorArchitecture: u16,
1178 lpMinimumApplicationAddress: *mut u8,
1179 lpMaximumApplicationAddress: *mut u8,
1180 dwActiveProcessorMask: *mut u8,
1181 dwNumberOfProcessors: u32,
1182 dwProcessorType: u32,
1183 dwAllocationGranularity: u32,
1184 wProcessorLevel: u16,
1185 wProcessorRevision: u16,
1188 fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32;
1191 let mut sysinfo = std::mem::zeroed();
1192 GetSystemInfo(&mut sysinfo);
1193 sysinfo.dwNumberOfProcessors as usize
1197 #[cfg(target_os = "redox")]
1198 fn num_cpus() -> usize {
1199 // FIXME: Implement num_cpus on Redox
1203 #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
1204 fn num_cpus() -> usize {
1208 #[cfg(any(target_os = "android",
1209 target_os = "cloudabi",
1210 target_os = "emscripten",
1211 target_os = "fuchsia",
1213 target_os = "linux",
1214 target_os = "macos",
1215 target_os = "solaris"))]
1216 fn num_cpus() -> usize {
1217 unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
1220 #[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "bitrig",
1221 target_os = "netbsd"))]
1222 fn num_cpus() -> usize {
1225 let mut cpus: libc::c_uint = 0;
1226 let mut cpus_size = std::mem::size_of_val(&cpus);
1229 cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint;
1232 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
1237 &mut cpus as *mut _ as *mut _,
1238 &mut cpus_size as *mut _ as *mut _,
1250 #[cfg(target_os = "openbsd")]
1251 fn num_cpus() -> usize {
1254 let mut cpus: libc::c_uint = 0;
1255 let mut cpus_size = std::mem::size_of_val(&cpus);
1256 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
1262 &mut cpus as *mut _ as *mut _,
1263 &mut cpus_size as *mut _ as *mut _,
1274 #[cfg(target_os = "haiku")]
1275 fn num_cpus() -> usize {
1281 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1282 let mut filtered = tests;
1284 // Remove tests that don't match the test filter
1285 filtered = match opts.filter {
1287 Some(ref filter) => {
1290 .filter(|test| if opts.filter_exact {
1291 test.desc.name.as_slice() == &filter[..]
1293 test.desc.name.as_slice().contains(&filter[..])
1299 // Skip tests that match any of the skip filters
1303 !opts.skip.iter().any(|sf| if opts.filter_exact {
1304 t.desc.name.as_slice() == &sf[..]
1306 t.desc.name.as_slice().contains(&sf[..])
1311 // Maybe pull out the ignored test and unignore them
1312 filtered = if !opts.run_ignored {
1315 fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
1316 if test.desc.ignore {
1317 let TestDescAndFn { desc, testfn } = test;
1318 Some(TestDescAndFn {
1329 filtered.into_iter().filter_map(filter).collect()
1332 // Sort the tests alphabetically
1333 filtered.sort_by(|t1, t2| {
1334 t1.desc.name.as_slice().cmp(t2.desc.name.as_slice())
1340 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1341 // convert benchmarks to tests, if we're not benchmarking them
1342 tests.into_iter().map(|x| {
1343 let testfn = match x.testfn {
1344 DynBenchFn(bench) => {
1345 DynTestFn(Box::new(move || {
1346 bench::run_once(|b| {
1347 __rust_begin_short_backtrace(|| bench.run(b))
1351 StaticBenchFn(benchfn) => {
1352 DynTestFn(Box::new(move || {
1353 bench::run_once(|b| {
1354 __rust_begin_short_backtrace(|| benchfn(b))
1371 test: TestDescAndFn,
1372 monitor_ch: Sender<MonitorMsg>,
1375 let TestDescAndFn { desc, testfn } = test;
1377 let ignore_because_panic_abort = cfg!(target_arch = "wasm32") &&
1378 !cfg!(target_os = "emscripten") &&
1379 desc.should_panic != ShouldPanic::No;
1381 if force_ignore || desc.ignore || ignore_because_panic_abort {
1382 monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap();
1386 fn run_test_inner(desc: TestDesc,
1387 monitor_ch: Sender<MonitorMsg>,
1389 testfn: Box<FnBox() + Send>) {
1390 // Buffer for capturing standard I/O
1391 let data = Arc::new(Mutex::new(Vec::new()));
1392 let data2 = data.clone();
1394 let name = desc.name.clone();
1395 let runtest = move || {
1396 let oldio = if !nocapture {
1398 io::set_print(Some(Box::new(Sink(data2.clone())))),
1399 io::set_panic(Some(Box::new(Sink(data2)))),
1405 let result = catch_unwind(AssertUnwindSafe(testfn));
1407 if let Some((printio, panicio)) = oldio {
1408 io::set_print(printio);
1409 io::set_panic(panicio);
1412 let test_result = calc_result(&desc, result);
1413 let stdout = data.lock().unwrap().to_vec();
1415 .send((desc.clone(), test_result, stdout))
1420 // If the platform is single-threaded we're just going to run
1421 // the test synchronously, regardless of the concurrency
1423 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
1424 if supports_threads {
1425 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
1426 cfg.spawn(runtest).unwrap();
1433 DynBenchFn(bencher) => {
1434 ::bench::benchmark(desc,
1437 |harness| bencher.run(harness));
1439 StaticBenchFn(benchfn) => {
1440 ::bench::benchmark(desc,
1443 |harness| (benchfn.clone())(harness));
1447 __rust_begin_short_backtrace(f)
1449 run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb))
1451 StaticTestFn(f) => {
1452 run_test_inner(desc, monitor_ch, opts.nocapture,
1453 Box::new(move || __rust_begin_short_backtrace(f)))
1458 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
1460 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
1464 fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> TestResult {
1465 match (&desc.should_panic, task_result) {
1466 (&ShouldPanic::No, Ok(())) |
1467 (&ShouldPanic::Yes, Err(_)) => TrOk,
1468 (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => {
1469 if err.downcast_ref::<String>()
1471 .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
1472 .map(|e| e.contains(msg))
1477 if desc.allow_fail {
1480 TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
1484 _ if desc.allow_fail => TrAllowedFail,
1489 #[derive(Clone, PartialEq)]
1490 pub struct MetricMap(BTreeMap<String, Metric>);
1493 pub fn new() -> MetricMap {
1494 MetricMap(BTreeMap::new())
1497 /// Insert a named `value` (+/- `noise`) metric into the map. The value
1498 /// must be non-negative. The `noise` indicates the uncertainty of the
1499 /// metric, which doubles as the "noise range" of acceptable
1500 /// pairwise-regressions on this named value, when comparing from one
1501 /// metric to the next using `compare_to_old`.
1503 /// If `noise` is positive, then it means this metric is of a value
1504 /// you want to see grow smaller, so a change larger than `noise` in the
1505 /// positive direction represents a regression.
1507 /// If `noise` is negative, then it means this metric is of a value
1508 /// you want to see grow larger, so a change larger than `noise` in the
1509 /// negative direction represents a regression.
1510 pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) {
1511 let m = Metric { value, noise };
1512 self.0.insert(name.to_owned(), m);
1515 pub fn fmt_metrics(&self) -> String {
1518 .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
1519 .collect::<Vec<_>>();
1527 /// A function that is opaque to the optimizer, to allow benchmarks to
1528 /// pretend to use outputs to assist in avoiding dead-code
1531 /// This function is a no-op, and does not even read from `dummy`.
1532 #[cfg(not(any(target_arch = "asmjs", target_arch = "wasm32")))]
1533 pub fn black_box<T>(dummy: T) -> T {
1534 // we need to "use" the argument in some way LLVM can't
1536 unsafe { asm!("" : : "r"(&dummy)) }
1539 #[cfg(any(target_arch = "asmjs", target_arch = "wasm32"))]
1541 pub fn black_box<T>(dummy: T) -> T {
1547 /// Callback for benchmark functions to run in their body.
1548 pub fn iter<T, F>(&mut self, mut inner: F)
1552 if self.mode == BenchMode::Single {
1553 ns_iter_inner(&mut inner, 1);
1557 self.summary = Some(iter(&mut inner));
1560 pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary>
1562 F: FnMut(&mut Bencher),
1565 return self.summary;
1569 fn ns_from_dur(dur: Duration) -> u64 {
1570 dur.as_secs() * 1_000_000_000 + (dur.subsec_nanos() as u64)
1573 fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64
1577 let start = Instant::now();
1581 return ns_from_dur(start.elapsed());
1585 pub fn iter<T, F>(inner: &mut F) -> stats::Summary
1589 // Initial bench run to get ballpark figure.
1590 let ns_single = ns_iter_inner(inner, 1);
1592 // Try to estimate iter count for 1ms falling back to 1m
1593 // iterations if first run took < 1ns.
1594 let ns_target_total = 1_000_000; // 1ms
1595 let mut n = ns_target_total / cmp::max(1, ns_single);
1597 // if the first run took more than 1ms we don't want to just
1598 // be left doing 0 iterations on every loop. The unfortunate
1599 // side effect of not being able to do as many runs is
1600 // automatically handled by the statistical analysis below
1601 // (i.e. larger error bars).
1604 let mut total_run = Duration::new(0, 0);
1605 let samples: &mut [f64] = &mut [0.0_f64; 50];
1607 let loop_start = Instant::now();
1609 for p in &mut *samples {
1610 *p = ns_iter_inner(inner, n) as f64 / n as f64;
1613 stats::winsorize(samples, 5.0);
1614 let summ = stats::Summary::new(samples);
1616 for p in &mut *samples {
1617 let ns = ns_iter_inner(inner, 5 * n);
1618 *p = ns as f64 / (5 * n) as f64;
1621 stats::winsorize(samples, 5.0);
1622 let summ5 = stats::Summary::new(samples);
1624 let loop_run = loop_start.elapsed();
1626 // If we've run for 100ms and seem to have converged to a
1628 if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
1629 summ.median - summ5.median < summ5.median_abs_dev
1634 total_run = total_run + loop_run;
1635 // Longest we ever run for is 3s.
1636 if total_run > Duration::from_secs(3) {
1640 // If we overflow here just return the results so far. We check a
1641 // multiplier of 10 because we're about to multiply by 2 and the
1642 // next iteration of the loop will also multiply by 5 (to calculate
1643 // the summ5 result)
1644 n = match n.checked_mul(10) {
1654 use std::panic::{catch_unwind, AssertUnwindSafe};
1657 use std::sync::{Arc, Mutex};
1659 use super::{Bencher, BenchSamples, BenchMode, Sink, MonitorMsg, TestDesc, Sender, TestResult};
1661 pub fn benchmark<F>(desc: TestDesc, monitor_ch: Sender<MonitorMsg>, nocapture: bool, f: F)
1663 F: FnMut(&mut Bencher),
1665 let mut bs = Bencher {
1666 mode: BenchMode::Auto,
1671 let data = Arc::new(Mutex::new(Vec::new()));
1672 let data2 = data.clone();
1674 let oldio = if !nocapture {
1676 io::set_print(Some(Box::new(Sink(data2.clone())))),
1677 io::set_panic(Some(Box::new(Sink(data2)))),
1683 let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f)));
1685 if let Some((printio, panicio)) = oldio {
1686 io::set_print(printio);
1687 io::set_panic(panicio);
1690 let test_result = match result { //bs.bench(f) {
1691 Ok(Some(ns_iter_summ)) => {
1692 let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
1693 let mb_s = bs.bytes * 1000 / ns_iter;
1695 let bs = BenchSamples {
1697 mb_s: mb_s as usize,
1699 TestResult::TrBench(bs)
1702 // iter not called, so no data.
1703 // FIXME: error in this case?
1704 let samples: &mut [f64] = &mut [0.0_f64; 1];
1705 let bs = BenchSamples {
1706 ns_iter_summ: stats::Summary::new(samples),
1709 TestResult::TrBench(bs)
1712 TestResult::TrFailed
1716 let stdout = data.lock().unwrap().to_vec();
1717 monitor_ch.send((desc, test_result, stdout)).unwrap();
1720 pub fn run_once<F>(f: F)
1722 F: FnMut(&mut Bencher),
1724 let mut bs = Bencher {
1725 mode: BenchMode::Single,
1735 use test::{TrFailed, TrFailedMsg, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc,
1736 TestDescAndFn, TestOpts, run_test, MetricMap, StaticTestName, DynTestName,
1737 DynTestFn, ShouldPanic};
1738 use std::sync::mpsc::channel;
1743 pub fn do_not_run_ignored_tests() {
1747 let desc = TestDescAndFn {
1749 name: StaticTestName("whatever"),
1751 should_panic: ShouldPanic::No,
1754 testfn: DynTestFn(Box::new(f)),
1756 let (tx, rx) = channel();
1757 run_test(&TestOpts::new(), false, desc, tx);
1758 let (_, res, _) = rx.recv().unwrap();
1759 assert!(res != TrOk);
1763 pub fn ignored_tests_result_in_ignored() {
1765 let desc = TestDescAndFn {
1767 name: StaticTestName("whatever"),
1769 should_panic: ShouldPanic::No,
1772 testfn: DynTestFn(Box::new(f)),
1774 let (tx, rx) = channel();
1775 run_test(&TestOpts::new(), false, desc, tx);
1776 let (_, res, _) = rx.recv().unwrap();
1777 assert!(res == TrIgnored);
1781 fn test_should_panic() {
1785 let desc = TestDescAndFn {
1787 name: StaticTestName("whatever"),
1789 should_panic: ShouldPanic::Yes,
1792 testfn: DynTestFn(Box::new(f)),
1794 let (tx, rx) = channel();
1795 run_test(&TestOpts::new(), false, desc, tx);
1796 let (_, res, _) = rx.recv().unwrap();
1797 assert!(res == TrOk);
1801 fn test_should_panic_good_message() {
1803 panic!("an error message");
1805 let desc = TestDescAndFn {
1807 name: StaticTestName("whatever"),
1809 should_panic: ShouldPanic::YesWithMessage("error message"),
1812 testfn: DynTestFn(Box::new(f)),
1814 let (tx, rx) = channel();
1815 run_test(&TestOpts::new(), false, desc, tx);
1816 let (_, res, _) = rx.recv().unwrap();
1817 assert!(res == TrOk);
1821 fn test_should_panic_bad_message() {
1823 panic!("an error message");
1825 let expected = "foobar";
1826 let failed_msg = "Panic did not include expected string";
1827 let desc = TestDescAndFn {
1829 name: StaticTestName("whatever"),
1831 should_panic: ShouldPanic::YesWithMessage(expected),
1834 testfn: DynTestFn(Box::new(f)),
1836 let (tx, rx) = channel();
1837 run_test(&TestOpts::new(), false, desc, tx);
1838 let (_, res, _) = rx.recv().unwrap();
1839 assert!(res == TrFailedMsg(format!("{} '{}'", failed_msg, expected)));
1843 fn test_should_panic_but_succeeds() {
1845 let desc = TestDescAndFn {
1847 name: StaticTestName("whatever"),
1849 should_panic: ShouldPanic::Yes,
1852 testfn: DynTestFn(Box::new(f)),
1854 let (tx, rx) = channel();
1855 run_test(&TestOpts::new(), false, desc, tx);
1856 let (_, res, _) = rx.recv().unwrap();
1857 assert!(res == TrFailed);
1861 fn parse_ignored_flag() {
1863 "progname".to_string(),
1864 "filter".to_string(),
1865 "--ignored".to_string(),
1867 let opts = match parse_opts(&args) {
1869 _ => panic!("Malformed arg in parse_ignored_flag"),
1871 assert!((opts.run_ignored));
1875 pub fn filter_for_ignored_option() {
1876 // When we run ignored tests the test filter should filter out all the
1877 // unignored tests and flip the ignore flag on the rest to false
1879 let mut opts = TestOpts::new();
1880 opts.run_tests = true;
1881 opts.run_ignored = true;
1884 vec![TestDescAndFn {
1886 name: StaticTestName("1"),
1888 should_panic: ShouldPanic::No,
1891 testfn: DynTestFn(Box::new(move || {})),
1895 name: StaticTestName("2"),
1897 should_panic: ShouldPanic::No,
1900 testfn: DynTestFn(Box::new(move || {})),
1902 let filtered = filter_tests(&opts, tests);
1904 assert_eq!(filtered.len(), 1);
1905 assert_eq!(filtered[0].desc.name.to_string(), "1");
1906 assert!(!filtered[0].desc.ignore);
1910 pub fn exact_filter_match() {
1911 fn tests() -> Vec<TestDescAndFn> {
1912 vec!["base", "base::test", "base::test1", "base::test2"]
1917 name: StaticTestName(name),
1919 should_panic: ShouldPanic::No,
1922 testfn: DynTestFn(Box::new(move || {}))
1927 let substr = filter_tests(
1929 filter: Some("base".into()),
1934 assert_eq!(substr.len(), 4);
1936 let substr = filter_tests(
1938 filter: Some("bas".into()),
1943 assert_eq!(substr.len(), 4);
1945 let substr = filter_tests(
1947 filter: Some("::test".into()),
1952 assert_eq!(substr.len(), 3);
1954 let substr = filter_tests(
1956 filter: Some("base::test".into()),
1961 assert_eq!(substr.len(), 3);
1963 let exact = filter_tests(
1965 filter: Some("base".into()),
1971 assert_eq!(exact.len(), 1);
1973 let exact = filter_tests(
1975 filter: Some("bas".into()),
1981 assert_eq!(exact.len(), 0);
1983 let exact = filter_tests(
1985 filter: Some("::test".into()),
1991 assert_eq!(exact.len(), 0);
1993 let exact = filter_tests(
1995 filter: Some("base::test".into()),
2001 assert_eq!(exact.len(), 1);
2005 pub fn sort_tests() {
2006 let mut opts = TestOpts::new();
2007 opts.run_tests = true;
2010 "sha1::test".to_string(),
2011 "isize::test_to_str".to_string(),
2012 "isize::test_pow".to_string(),
2013 "test::do_not_run_ignored_tests".to_string(),
2014 "test::ignored_tests_result_in_ignored".to_string(),
2015 "test::first_free_arg_should_be_a_filter".to_string(),
2016 "test::parse_ignored_flag".to_string(),
2017 "test::filter_for_ignored_option".to_string(),
2018 "test::sort_tests".to_string(),
2022 let mut tests = Vec::new();
2023 for name in &names {
2024 let test = TestDescAndFn {
2026 name: DynTestName((*name).clone()),
2028 should_panic: ShouldPanic::No,
2031 testfn: DynTestFn(Box::new(testfn)),
2037 let filtered = filter_tests(&opts, tests);
2039 let expected = vec![
2040 "isize::test_pow".to_string(),
2041 "isize::test_to_str".to_string(),
2042 "sha1::test".to_string(),
2043 "test::do_not_run_ignored_tests".to_string(),
2044 "test::filter_for_ignored_option".to_string(),
2045 "test::first_free_arg_should_be_a_filter".to_string(),
2046 "test::ignored_tests_result_in_ignored".to_string(),
2047 "test::parse_ignored_flag".to_string(),
2048 "test::sort_tests".to_string(),
2051 for (a, b) in expected.iter().zip(filtered) {
2052 assert!(*a == b.desc.name.to_string());
2057 pub fn test_metricmap_compare() {
2058 let mut m1 = MetricMap::new();
2059 let mut m2 = MetricMap::new();
2060 m1.insert_metric("in-both-noise", 1000.0, 200.0);
2061 m2.insert_metric("in-both-noise", 1100.0, 200.0);
2063 m1.insert_metric("in-first-noise", 1000.0, 2.0);
2064 m2.insert_metric("in-second-noise", 1000.0, 2.0);
2066 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
2067 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
2069 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
2070 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
2072 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
2073 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
2075 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
2076 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
2080 pub fn test_bench_once_no_iter() {
2081 fn f(_: &mut Bencher) {}
2086 pub fn test_bench_once_iter() {
2087 fn f(b: &mut Bencher) {
2094 pub fn test_bench_no_iter() {
2095 fn f(_: &mut Bencher) {}
2097 let (tx, rx) = channel();
2099 let desc = TestDesc {
2100 name: StaticTestName("f"),
2102 should_panic: ShouldPanic::No,
2106 ::bench::benchmark(desc,
2114 pub fn test_bench_iter() {
2115 fn f(b: &mut Bencher) {
2119 let (tx, rx) = channel();
2121 let desc = TestDesc {
2122 name: StaticTestName("f"),
2124 should_panic: ShouldPanic::No,
2128 ::bench::benchmark(desc,