1 // Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Support code for rustc's built in unit-test and micro-benchmarking
14 //! Almost all user code will only be interested in `Bencher` and
15 //! `black_box`. All other interactions (such as writing tests and
16 //! benchmarks themselves) should be done via the `#[test]` and
17 //! `#[bench]` attributes.
19 //! See the [Testing Chapter](../book/first-edition/testing.html) of the book for more details.
21 // Currently, not much of this is meant for users. It is intended to
22 // support the simplest interface possible for representing and
23 // running tests while providing a base that other test frameworks may
26 // NB: this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to
27 // this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
28 // cargo) to detect this crate.
29 #![crate_name = "test"]
30 #![unstable(feature = "test", issue = "27812")]
31 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
32 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
33 html_root_url = "https://doc.rust-lang.org/nightly/",
34 test(attr(deny(warnings))))]
39 #![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
40 #![feature(set_stdio)]
41 #![feature(panic_unwind)]
42 #![feature(staged_api)]
46 #[cfg(any(unix, target_os = "cloudabi"))]
48 extern crate panic_unwind;
50 pub use self::TestFn::*;
51 pub use self::ColorConfig::*;
52 pub use self::TestResult::*;
53 pub use self::TestName::*;
54 use self::TestEvent::*;
55 use self::NamePadding::*;
56 use self::OutputLocation::*;
58 use std::panic::{catch_unwind, AssertUnwindSafe};
60 use std::boxed::FnBox;
62 use std::collections::BTreeMap;
66 use std::io::prelude::*;
68 use std::iter::repeat;
69 use std::path::PathBuf;
70 use std::sync::mpsc::{channel, Sender};
71 use std::sync::{Arc, Mutex};
73 use std::time::{Instant, Duration};
77 const TEST_WARN_TIMEOUT_S: u64 = 60;
78 const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode
80 // to be used by rustc to compile tests in libtest
82 pub use {Bencher, TestName, TestResult, TestDesc, TestDescAndFn, TestOpts, TrFailed,
83 TrFailedMsg, TrIgnored, TrOk, Metric, MetricMap, StaticTestFn, StaticTestName,
84 DynTestName, DynTestFn, run_test, test_main, test_main_static, filter_tests,
85 parse_opts, StaticBenchFn, ShouldPanic, Options};
91 use formatters::{OutputFormatter, PrettyFormatter, TerseFormatter, JsonFormatter};
93 // The name of a test. By convention this follows the rules for rust
94 // paths; i.e. it should be a series of identifiers separated by double
95 // colons. This way if some test runner wants to arrange the tests
96 // hierarchically it may.
98 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
100 StaticTestName(&'static str),
102 AlignedTestName(Cow<'static, str>, NamePadding),
105 fn as_slice(&self) -> &str {
107 StaticTestName(s) => s,
108 DynTestName(ref s) => s,
109 AlignedTestName(ref s, _) => &*s,
113 fn padding(&self) -> NamePadding {
115 &AlignedTestName(_, p) => p,
120 fn with_padding(&self, padding: NamePadding) -> TestName {
121 let name = match self {
122 &TestName::StaticTestName(name) => Cow::Borrowed(name),
123 &TestName::DynTestName(ref name) => Cow::Owned(name.clone()),
124 &TestName::AlignedTestName(ref name, _) => name.clone(),
127 TestName::AlignedTestName(name, padding)
130 impl fmt::Display for TestName {
131 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
132 fmt::Display::fmt(self.as_slice(), f)
136 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
137 pub enum NamePadding {
143 fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
144 let mut name = String::from(self.name.as_slice());
145 let fill = column_count.saturating_sub(name.len());
146 let pad = repeat(" ").take(fill).collect::<String>();
157 /// Represents a benchmark function.
158 pub trait TDynBenchFn: Send {
159 fn run(&self, harness: &mut Bencher);
162 // A function that runs a test. If the function returns successfully,
163 // the test succeeds; if the function panics then the test fails. We
164 // may need to come up with a more clever definition of test in order
165 // to support isolation of tests into threads.
168 StaticBenchFn(fn(&mut Bencher)),
169 DynTestFn(Box<FnBox() + Send>),
170 DynBenchFn(Box<TDynBenchFn + 'static>),
174 fn padding(&self) -> NamePadding {
176 StaticTestFn(..) => PadNone,
177 StaticBenchFn(..) => PadOnRight,
178 DynTestFn(..) => PadNone,
179 DynBenchFn(..) => PadOnRight,
184 impl fmt::Debug for TestFn {
185 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
186 f.write_str(match *self {
187 StaticTestFn(..) => "StaticTestFn(..)",
188 StaticBenchFn(..) => "StaticBenchFn(..)",
189 DynTestFn(..) => "DynTestFn(..)",
190 DynBenchFn(..) => "DynBenchFn(..)",
195 /// Manager of the benchmarking runs.
197 /// This is fed into functions marked with `#[bench]` to allow for
198 /// set-up & tear-down before running a piece of code repeatedly via a
203 summary: Option<stats::Summary>,
207 #[derive(Clone, PartialEq, Eq)]
213 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
214 pub enum ShouldPanic {
217 YesWithMessage(&'static str),
220 // The definition of a single test. A test runner will run a list of
222 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
223 pub struct TestDesc {
226 pub should_panic: ShouldPanic,
227 pub allow_fail: bool,
231 pub struct TestDescAndFn {
236 #[derive(Clone, PartialEq, Debug, Copy)]
243 pub fn new(value: f64, noise: f64) -> Metric {
244 Metric { value, noise }
248 /// In case we want to add other options as well, just add them in this struct.
249 #[derive(Copy, Clone, Debug)]
251 display_output: bool,
255 pub fn new() -> Options {
256 Options { display_output: false }
259 pub fn display_output(mut self, display_output: bool) -> Options {
260 self.display_output = display_output;
265 // The default console test runner. It accepts the command line
266 // arguments and a vector of test_descs.
267 pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) {
268 let mut opts = match parse_opts(args) {
271 eprintln!("error: {}", msg);
277 opts.options = options;
279 if let Err(e) = list_tests_console(&opts, tests) {
280 eprintln!("error: io error when listing tests: {:?}", e);
284 match run_tests_console(&opts, tests) {
286 Ok(false) => process::exit(101),
288 eprintln!("error: io error when listing tests: {:?}", e);
295 // A variant optimized for invocation with a static test vector.
296 // This will panic (intentionally) when fed any dynamic tests, because
297 // it is copying the static values out into a dynamic vector and cannot
298 // copy dynamic values. It is doing this because from this point on
299 // a Vec<TestDescAndFn> is used in order to effect ownership-transfer
300 // semantics into parallel test runners, which in turn requires a Vec<>
301 // rather than a &[].
302 pub fn test_main_static(tests: &[TestDescAndFn]) {
303 let args = env::args().collect::<Vec<_>>();
304 let owned_tests = tests
306 .map(|t| match t.testfn {
309 testfn: StaticTestFn(f),
310 desc: t.desc.clone(),
313 StaticBenchFn(f) => {
315 testfn: StaticBenchFn(f),
316 desc: t.desc.clone(),
319 _ => panic!("non-static tests passed to test::test_main_static"),
322 test_main(&args, owned_tests, Options::new())
325 #[derive(Copy, Clone, Debug)]
326 pub enum ColorConfig {
332 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
333 pub enum OutputFormat {
340 pub struct TestOpts {
342 pub filter: Option<String>,
343 pub filter_exact: bool,
344 pub run_ignored: bool,
346 pub bench_benchmarks: bool,
347 pub logfile: Option<PathBuf>,
349 pub color: ColorConfig,
350 pub format: OutputFormat,
351 pub test_threads: Option<usize>,
352 pub skip: Vec<String>,
353 pub options: Options,
358 fn new() -> TestOpts {
365 bench_benchmarks: false,
369 format: OutputFormat::Pretty,
372 options: Options::new(),
377 /// Result of parsing the options.
378 pub type OptRes = Result<TestOpts, String>;
380 fn optgroups() -> getopts::Options {
381 let mut opts = getopts::Options::new();
382 opts.optflag("", "ignored", "Run ignored tests")
383 .optflag("", "test", "Run tests and not benchmarks")
384 .optflag("", "bench", "Run benchmarks instead of tests")
385 .optflag("", "list", "List all tests and benchmarks")
386 .optflag("h", "help", "Display this message (longer with --help)")
390 "Write logs to the specified file instead \
397 "don't capture stdout/stderr of each \
398 task, allow printing directly",
403 "Number of threads used for running tests \
410 "Skip tests whose names contain FILTER (this flag can \
411 be used multiple times)",
417 "Display one character per test instead of one line. \
418 Alias to --format=terse",
423 "Exactly match filters rather than by substring",
428 "Configure coloring of output:
429 auto = colorize if stdout is a tty and tests are run on serially (default);
430 always = always colorize output;
431 never = never colorize output;",
437 "Configure formatting of output:
438 pretty = Print verbose output;
439 terse = Display one character per test;
440 json = Output a json document",
446 "Enable nightly-only flags:
447 unstable-options = Allow use of experimental features",
453 fn usage(binary: &str, options: &getopts::Options) {
454 let message = format!("Usage: {} [OPTIONS] [FILTER]", binary);
458 The FILTER string is tested against the name of all tests, and only those
459 tests whose names contain the filter are run.
461 By default, all tests are run in parallel. This can be altered with the
462 --test-threads flag or the RUST_TEST_THREADS environment variable when running
465 All tests have their standard output and standard error captured by default.
466 This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE
467 environment variable to a value other than "0". Logging is not captured by default.
471 #[test] - Indicates a function is a test to be run. This function
473 #[bench] - Indicates a function is a benchmark to be run. This
474 function takes one argument (test::Bencher).
475 #[should_panic] - This function (also labeled with #[test]) will only pass if
476 the code causes a panic (an assertion failure or panic!)
477 A message may be provided, which the failure string must
478 contain: #[should_panic(expected = "foo")].
479 #[ignore] - When applied to a function which is already attributed as a
480 test, then the test runner will ignore these tests during
481 normal test runs. Running with --ignored will run these
483 usage = options.usage(&message)
487 // FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566
488 fn is_nightly() -> bool {
489 // Whether this is a feature-staged build, i.e. on the beta or stable channel
490 let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some();
491 // Whether we should enable unstable features for bootstrapping
492 let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok();
494 bootstrap || !disable_unstable_features
497 // Parses command line arguments into test options
498 pub fn parse_opts(args: &[String]) -> Option<OptRes> {
499 let mut allow_unstable = false;
500 let opts = optgroups();
501 let args = args.get(1..).unwrap_or(args);
502 let matches = match opts.parse(args) {
504 Err(f) => return Some(Err(f.to_string())),
507 if let Some(opt) = matches.opt_str("Z") {
510 "the option `Z` is only accepted on the nightly compiler"
516 "unstable-options" => {
517 allow_unstable = true;
520 return Some(Err("Unrecognized option to `Z`".into()));
525 if matches.opt_present("h") {
526 usage(&args[0], &opts);
530 let filter = if !matches.free.is_empty() {
531 Some(matches.free[0].clone())
536 let run_ignored = matches.opt_present("ignored");
537 let quiet = matches.opt_present("quiet");
538 let exact = matches.opt_present("exact");
539 let list = matches.opt_present("list");
541 let logfile = matches.opt_str("logfile");
542 let logfile = logfile.map(|s| PathBuf::from(&s));
544 let bench_benchmarks = matches.opt_present("bench");
545 let run_tests = !bench_benchmarks || matches.opt_present("test");
547 let mut nocapture = matches.opt_present("nocapture");
549 nocapture = match env::var("RUST_TEST_NOCAPTURE") {
550 Ok(val) => &val != "0",
555 let test_threads = match matches.opt_str("test-threads") {
557 match n_str.parse::<usize>() {
558 Ok(0) => return Some(Err(format!("argument for --test-threads must not be 0"))),
561 return Some(Err(format!(
562 "argument for --test-threads must be a number > 0 \
572 let color = match matches.opt_str("color").as_ref().map(|s| &**s) {
573 Some("auto") | None => AutoColor,
574 Some("always") => AlwaysColor,
575 Some("never") => NeverColor,
578 return Some(Err(format!(
579 "argument for --color must be auto, always, or never (was \
586 let format = match matches.opt_str("format").as_ref().map(|s| &**s) {
587 None if quiet => OutputFormat::Terse,
588 Some("pretty") | None => OutputFormat::Pretty,
589 Some("terse") => OutputFormat::Terse,
593 "The \"json\" format is only accepted on the nightly compiler"
601 return Some(Err(format!(
602 "argument for --format must be pretty, terse, or json (was \
609 let test_opts = TestOpts {
621 skip: matches.opt_strs("skip"),
622 options: Options::new(),
628 #[derive(Clone, PartialEq)]
629 pub struct BenchSamples {
630 ns_iter_summ: stats::Summary,
634 #[derive(Clone, PartialEq)]
635 pub enum TestResult {
641 TrBench(BenchSamples),
644 unsafe impl Send for TestResult {}
646 enum OutputLocation<T> {
647 Pretty(Box<term::StdoutTerminal>),
651 impl<T: Write> Write for OutputLocation<T> {
652 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
654 Pretty(ref mut term) => term.write(buf),
655 Raw(ref mut stdout) => stdout.write(buf),
659 fn flush(&mut self) -> io::Result<()> {
661 Pretty(ref mut term) => term.flush(),
662 Raw(ref mut stdout) => stdout.flush(),
667 struct ConsoleTestState {
668 log_out: Option<File>,
677 failures: Vec<(TestDesc, Vec<u8>)>,
678 not_failures: Vec<(TestDesc, Vec<u8>)>,
682 impl ConsoleTestState {
683 pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState> {
684 let log_out = match opts.logfile {
685 Some(ref path) => Some(File::create(path)?),
689 Ok(ConsoleTestState {
698 metrics: MetricMap::new(),
699 failures: Vec::new(),
700 not_failures: Vec::new(),
701 options: opts.options,
705 pub fn write_log<S: AsRef<str>>(&mut self, msg: S) -> io::Result<()> {
706 let msg = msg.as_ref();
709 Some(ref mut o) => o.write_all(msg.as_bytes()),
713 pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
714 self.write_log(format!(
717 TrOk => "ok".to_owned(),
718 TrFailed => "failed".to_owned(),
719 TrFailedMsg(ref msg) => format!("failed: {}", msg),
720 TrIgnored => "ignored".to_owned(),
721 TrAllowedFail => "failed (allowed)".to_owned(),
722 TrBench(ref bs) => fmt_bench_samples(bs),
728 fn current_test_count(&self) -> usize {
729 self.passed + self.failed + self.ignored + self.measured + self.allowed_fail
733 // Format a number with thousands separators
734 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
736 let mut output = String::new();
737 let mut trailing = false;
738 for &pow in &[9, 6, 3, 0] {
739 let base = 10_usize.pow(pow);
740 if pow == 0 || trailing || n / base != 0 {
742 output.write_fmt(format_args!("{}", n / base)).unwrap();
744 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
757 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
759 let mut output = String::new();
761 let median = bs.ns_iter_summ.median as usize;
762 let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
765 .write_fmt(format_args!(
766 "{:>11} ns/iter (+/- {})",
767 fmt_thousands_sep(median, ','),
768 fmt_thousands_sep(deviation, ',')
773 .write_fmt(format_args!(" = {} MB/s", bs.mb_s))
779 // List the tests to console, and optionally to logfile. Filters are honored.
780 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
781 let mut output = match term::stdout() {
782 None => Raw(io::stdout()),
783 Some(t) => Pretty(t),
786 let quiet = opts.format == OutputFormat::Terse;
787 let mut st = ConsoleTestState::new(opts)?;
792 for test in filter_tests(&opts, tests) {
796 desc: TestDesc { name, .. },
800 let fntype = match testfn {
801 StaticTestFn(..) | DynTestFn(..) => {
812 writeln!(output, "{}: {}", name, fntype)?;
813 st.write_log(format!("{} {}\n", fntype, name))?;
816 fn plural(count: u32, s: &str) -> String {
818 1 => format!("{} {}", 1, s),
819 n => format!("{} {}s", n, s),
824 if ntest != 0 || nbench != 0 {
825 writeln!(output, "")?;
830 plural(ntest, "test"),
831 plural(nbench, "benchmark")
838 // A simple console test runner
839 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
842 st: &mut ConsoleTestState,
843 out: &mut OutputFormatter,
844 ) -> io::Result<()> {
846 match (*event).clone() {
847 TeFiltered(ref filtered_tests) => {
848 st.total = filtered_tests.len();
849 out.write_run_start(filtered_tests.len())
851 TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out),
852 TeWait(ref test) => out.write_test_start(test),
853 TeTimeout(ref test) => out.write_timeout(test),
854 TeResult(test, result, stdout) => {
855 st.write_log_result(&test, &result)?;
856 out.write_result(&test, &result, &*stdout)?;
860 st.not_failures.push((test, stdout));
862 TrIgnored => st.ignored += 1,
863 TrAllowedFail => st.allowed_fail += 1,
865 st.metrics.insert_metric(
866 test.name.as_slice(),
867 bs.ns_iter_summ.median,
868 bs.ns_iter_summ.max - bs.ns_iter_summ.min,
874 st.failures.push((test, stdout));
876 TrFailedMsg(msg) => {
878 let mut stdout = stdout;
879 stdout.extend_from_slice(format!("note: {}", msg).as_bytes());
880 st.failures.push((test, stdout));
888 let output = match term::stdout() {
889 None => Raw(io::stdout()),
890 Some(t) => Pretty(t),
893 let max_name_len = tests
895 .max_by_key(|t| len_if_padded(*t))
896 .map(|t| t.desc.name.as_slice().len())
899 let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1;
901 let mut out: Box<OutputFormatter> = match opts.format {
902 OutputFormat::Pretty => Box::new(PrettyFormatter::new(
908 OutputFormat::Terse => Box::new(TerseFormatter::new(
914 OutputFormat::Json => Box::new(JsonFormatter::new(output)),
916 let mut st = ConsoleTestState::new(opts)?;
917 fn len_if_padded(t: &TestDescAndFn) -> usize {
918 match t.testfn.padding() {
920 PadOnRight => t.desc.name.as_slice().len(),
924 run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?;
926 assert!(st.current_test_count() == st.total);
928 return out.write_run_finish(&st);
932 fn should_sort_failures_before_printing_them() {
933 let test_a = TestDesc {
934 name: StaticTestName("a"),
936 should_panic: ShouldPanic::No,
940 let test_b = TestDesc {
941 name: StaticTestName("b"),
943 should_panic: ShouldPanic::No,
947 let mut out = PrettyFormatter::new(Raw(Vec::new()), false, 10, false);
949 let st = ConsoleTestState {
958 metrics: MetricMap::new(),
959 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
960 options: Options::new(),
961 not_failures: Vec::new(),
964 out.write_failures(&st).unwrap();
965 let s = match out.output_location() {
966 &Raw(ref m) => String::from_utf8_lossy(&m[..]),
967 &Pretty(_) => unreachable!(),
970 let apos = s.find("a").unwrap();
971 let bpos = s.find("b").unwrap();
972 assert!(apos < bpos);
975 fn use_color(opts: &TestOpts) -> bool {
977 AutoColor => !opts.nocapture && stdout_isatty(),
983 #[cfg(any(target_os = "cloudabi",
985 all(target_arch = "wasm32", not(target_os = "emscripten"))))]
986 fn stdout_isatty() -> bool {
987 // FIXME: Implement isatty on Redox
991 fn stdout_isatty() -> bool {
992 unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
995 fn stdout_isatty() -> bool {
998 type HANDLE = *mut u8;
999 type LPDWORD = *mut u32;
1000 const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD;
1002 fn GetStdHandle(which: DWORD) -> HANDLE;
1003 fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL;
1006 let handle = GetStdHandle(STD_OUTPUT_HANDLE);
1008 GetConsoleMode(handle, &mut out) != 0
1013 pub enum TestEvent {
1014 TeFiltered(Vec<TestDesc>),
1016 TeResult(TestDesc, TestResult, Vec<u8>),
1017 TeTimeout(TestDesc),
1018 TeFilteredOut(usize),
1021 pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>);
1023 struct Sink(Arc<Mutex<Vec<u8>>>);
1024 impl Write for Sink {
1025 fn write(&mut self, data: &[u8]) -> io::Result<usize> {
1026 Write::write(&mut *self.0.lock().unwrap(), data)
1028 fn flush(&mut self) -> io::Result<()> {
1033 pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
1035 F: FnMut(TestEvent) -> io::Result<()>,
1037 use std::collections::HashMap;
1038 use std::sync::mpsc::RecvTimeoutError;
1040 let tests_len = tests.len();
1042 let mut filtered_tests = filter_tests(opts, tests);
1043 if !opts.bench_benchmarks {
1044 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
1047 let filtered_tests = {
1048 let mut filtered_tests = filtered_tests;
1049 for test in filtered_tests.iter_mut() {
1050 test.desc.name = test.desc.name.with_padding(test.testfn.padding());
1056 let filtered_out = tests_len - filtered_tests.len();
1057 callback(TeFilteredOut(filtered_out))?;
1059 let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
1061 callback(TeFiltered(filtered_descs))?;
1063 let (filtered_tests, filtered_benchs): (Vec<_>, _) =
1064 filtered_tests.into_iter().partition(|e| match e.testfn {
1065 StaticTestFn(_) | DynTestFn(_) => true,
1069 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
1071 let mut remaining = filtered_tests;
1072 remaining.reverse();
1073 let mut pending = 0;
1075 let (tx, rx) = channel::<MonitorMsg>();
1077 let mut running_tests: HashMap<TestDesc, Instant> = HashMap::new();
1079 fn get_timed_out_tests(running_tests: &mut HashMap<TestDesc, Instant>) -> Vec<TestDesc> {
1080 let now = Instant::now();
1081 let timed_out = running_tests
1083 .filter_map(|(desc, timeout)| if &now >= timeout {
1089 for test in &timed_out {
1090 running_tests.remove(test);
1095 fn calc_timeout(running_tests: &HashMap<TestDesc, Instant>) -> Option<Duration> {
1096 running_tests.values().min().map(|next_timeout| {
1097 let now = Instant::now();
1098 if *next_timeout >= now {
1106 if concurrency == 1 {
1107 while !remaining.is_empty() {
1108 let test = remaining.pop().unwrap();
1109 callback(TeWait(test.desc.clone()))?;
1110 run_test(opts, !opts.run_tests, test, tx.clone());
1111 let (test, result, stdout) = rx.recv().unwrap();
1112 callback(TeResult(test, result, stdout))?;
1115 while pending > 0 || !remaining.is_empty() {
1116 while pending < concurrency && !remaining.is_empty() {
1117 let test = remaining.pop().unwrap();
1118 let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S);
1119 running_tests.insert(test.desc.clone(), timeout);
1120 callback(TeWait(test.desc.clone()))?; //here no pad
1121 run_test(opts, !opts.run_tests, test, tx.clone());
1127 if let Some(timeout) = calc_timeout(&running_tests) {
1128 res = rx.recv_timeout(timeout);
1129 for test in get_timed_out_tests(&mut running_tests) {
1130 callback(TeTimeout(test))?;
1132 if res != Err(RecvTimeoutError::Timeout) {
1136 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
1141 let (desc, result, stdout) = res.unwrap();
1142 running_tests.remove(&desc);
1144 callback(TeResult(desc, result, stdout))?;
1149 if opts.bench_benchmarks {
1150 // All benchmarks run at the end, in serial.
1151 for b in filtered_benchs {
1152 callback(TeWait(b.desc.clone()))?;
1153 run_test(opts, false, b, tx.clone());
1154 let (test, result, stdout) = rx.recv().unwrap();
1155 callback(TeResult(test, result, stdout))?;
1161 #[allow(deprecated)]
1162 fn get_concurrency() -> usize {
1163 return match env::var("RUST_TEST_THREADS") {
1165 let opt_n: Option<usize> = s.parse().ok();
1167 Some(n) if n > 0 => n,
1170 "RUST_TEST_THREADS is `{}`, should be a positive integer.",
1176 Err(..) => num_cpus(),
1181 fn num_cpus() -> usize {
1183 struct SYSTEM_INFO {
1184 wProcessorArchitecture: u16,
1187 lpMinimumApplicationAddress: *mut u8,
1188 lpMaximumApplicationAddress: *mut u8,
1189 dwActiveProcessorMask: *mut u8,
1190 dwNumberOfProcessors: u32,
1191 dwProcessorType: u32,
1192 dwAllocationGranularity: u32,
1193 wProcessorLevel: u16,
1194 wProcessorRevision: u16,
1197 fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32;
1200 let mut sysinfo = std::mem::zeroed();
1201 GetSystemInfo(&mut sysinfo);
1202 sysinfo.dwNumberOfProcessors as usize
1206 #[cfg(target_os = "redox")]
1207 fn num_cpus() -> usize {
1208 // FIXME: Implement num_cpus on Redox
1212 #[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
1213 fn num_cpus() -> usize {
1217 #[cfg(any(target_os = "android",
1218 target_os = "cloudabi",
1219 target_os = "emscripten",
1220 target_os = "fuchsia",
1222 target_os = "linux",
1223 target_os = "macos",
1224 target_os = "solaris"))]
1225 fn num_cpus() -> usize {
1226 unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize }
1229 #[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "bitrig",
1230 target_os = "netbsd"))]
1231 fn num_cpus() -> usize {
1234 let mut cpus: libc::c_uint = 0;
1235 let mut cpus_size = std::mem::size_of_val(&cpus);
1238 cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint;
1241 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
1246 &mut cpus as *mut _ as *mut _,
1247 &mut cpus_size as *mut _ as *mut _,
1259 #[cfg(target_os = "openbsd")]
1260 fn num_cpus() -> usize {
1263 let mut cpus: libc::c_uint = 0;
1264 let mut cpus_size = std::mem::size_of_val(&cpus);
1265 let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0];
1271 &mut cpus as *mut _ as *mut _,
1272 &mut cpus_size as *mut _ as *mut _,
1283 #[cfg(target_os = "haiku")]
1284 fn num_cpus() -> usize {
1290 pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1291 let mut filtered = tests;
1293 // Remove tests that don't match the test filter
1294 filtered = match opts.filter {
1296 Some(ref filter) => {
1299 .filter(|test| if opts.filter_exact {
1300 test.desc.name.as_slice() == &filter[..]
1302 test.desc.name.as_slice().contains(&filter[..])
1308 // Skip tests that match any of the skip filters
1312 !opts.skip.iter().any(|sf| if opts.filter_exact {
1313 t.desc.name.as_slice() == &sf[..]
1315 t.desc.name.as_slice().contains(&sf[..])
1320 // Maybe pull out the ignored test and unignore them
1321 filtered = if !opts.run_ignored {
1324 fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
1325 if test.desc.ignore {
1326 let TestDescAndFn { desc, testfn } = test;
1327 Some(TestDescAndFn {
1338 filtered.into_iter().filter_map(filter).collect()
1341 // Sort the tests alphabetically
1342 filtered.sort_by(|t1, t2| {
1343 t1.desc.name.as_slice().cmp(t2.desc.name.as_slice())
1349 pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
1350 // convert benchmarks to tests, if we're not benchmarking them
1351 tests.into_iter().map(|x| {
1352 let testfn = match x.testfn {
1353 DynBenchFn(bench) => {
1354 DynTestFn(Box::new(move || {
1355 bench::run_once(|b| {
1356 __rust_begin_short_backtrace(|| bench.run(b))
1360 StaticBenchFn(benchfn) => {
1361 DynTestFn(Box::new(move || {
1362 bench::run_once(|b| {
1363 __rust_begin_short_backtrace(|| benchfn(b))
1380 test: TestDescAndFn,
1381 monitor_ch: Sender<MonitorMsg>,
1384 let TestDescAndFn { desc, testfn } = test;
1386 let ignore_because_panic_abort = cfg!(target_arch = "wasm32") &&
1387 !cfg!(target_os = "emscripten") &&
1388 desc.should_panic != ShouldPanic::No;
1390 if force_ignore || desc.ignore || ignore_because_panic_abort {
1391 monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap();
1395 fn run_test_inner(desc: TestDesc,
1396 monitor_ch: Sender<MonitorMsg>,
1398 testfn: Box<FnBox() + Send>) {
1399 // Buffer for capturing standard I/O
1400 let data = Arc::new(Mutex::new(Vec::new()));
1401 let data2 = data.clone();
1403 let name = desc.name.clone();
1404 let runtest = move || {
1405 let oldio = if !nocapture {
1407 io::set_print(Some(Box::new(Sink(data2.clone())))),
1408 io::set_panic(Some(Box::new(Sink(data2)))),
1414 let result = catch_unwind(AssertUnwindSafe(testfn));
1416 if let Some((printio, panicio)) = oldio {
1417 io::set_print(printio);
1418 io::set_panic(panicio);
1421 let test_result = calc_result(&desc, result);
1422 let stdout = data.lock().unwrap().to_vec();
1424 .send((desc.clone(), test_result, stdout))
1429 // If the platform is single-threaded we're just going to run
1430 // the test synchronously, regardless of the concurrency
1432 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
1433 if supports_threads {
1434 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
1435 cfg.spawn(runtest).unwrap();
1442 DynBenchFn(bencher) => {
1443 ::bench::benchmark(desc,
1446 |harness| bencher.run(harness));
1448 StaticBenchFn(benchfn) => {
1449 ::bench::benchmark(desc,
1452 |harness| (benchfn.clone())(harness));
1456 __rust_begin_short_backtrace(f)
1458 run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb))
1460 StaticTestFn(f) => {
1461 run_test_inner(desc, monitor_ch, opts.nocapture,
1462 Box::new(move || __rust_begin_short_backtrace(f)))
1467 /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
1469 fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
1473 fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> TestResult {
1474 match (&desc.should_panic, task_result) {
1475 (&ShouldPanic::No, Ok(())) |
1476 (&ShouldPanic::Yes, Err(_)) => TrOk,
1477 (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => {
1478 if err.downcast_ref::<String>()
1480 .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e))
1481 .map(|e| e.contains(msg))
1486 if desc.allow_fail {
1489 TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
1493 _ if desc.allow_fail => TrAllowedFail,
1498 #[derive(Clone, PartialEq)]
1499 pub struct MetricMap(BTreeMap<String, Metric>);
1502 pub fn new() -> MetricMap {
1503 MetricMap(BTreeMap::new())
1506 /// Insert a named `value` (+/- `noise`) metric into the map. The value
1507 /// must be non-negative. The `noise` indicates the uncertainty of the
1508 /// metric, which doubles as the "noise range" of acceptable
1509 /// pairwise-regressions on this named value, when comparing from one
1510 /// metric to the next using `compare_to_old`.
1512 /// If `noise` is positive, then it means this metric is of a value
1513 /// you want to see grow smaller, so a change larger than `noise` in the
1514 /// positive direction represents a regression.
1516 /// If `noise` is negative, then it means this metric is of a value
1517 /// you want to see grow larger, so a change larger than `noise` in the
1518 /// negative direction represents a regression.
1519 pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) {
1520 let m = Metric { value, noise };
1521 self.0.insert(name.to_owned(), m);
1524 pub fn fmt_metrics(&self) -> String {
1527 .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise))
1528 .collect::<Vec<_>>();
1536 /// A function that is opaque to the optimizer, to allow benchmarks to
1537 /// pretend to use outputs to assist in avoiding dead-code
1540 /// This function is a no-op, and does not even read from `dummy`.
1541 #[cfg(not(any(target_arch = "asmjs", target_arch = "wasm32")))]
1542 pub fn black_box<T>(dummy: T) -> T {
1543 // we need to "use" the argument in some way LLVM can't
1545 unsafe { asm!("" : : "r"(&dummy)) }
1548 #[cfg(any(target_arch = "asmjs", target_arch = "wasm32"))]
1550 pub fn black_box<T>(dummy: T) -> T {
1556 /// Callback for benchmark functions to run in their body.
1557 pub fn iter<T, F>(&mut self, mut inner: F)
1561 if self.mode == BenchMode::Single {
1562 ns_iter_inner(&mut inner, 1);
1566 self.summary = Some(iter(&mut inner));
1569 pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary>
1571 F: FnMut(&mut Bencher),
1574 return self.summary;
1578 fn ns_from_dur(dur: Duration) -> u64 {
1579 dur.as_secs() * 1_000_000_000 + (dur.subsec_nanos() as u64)
1582 fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64
1586 let start = Instant::now();
1590 return ns_from_dur(start.elapsed());
1594 pub fn iter<T, F>(inner: &mut F) -> stats::Summary
1598 // Initial bench run to get ballpark figure.
1599 let ns_single = ns_iter_inner(inner, 1);
1601 // Try to estimate iter count for 1ms falling back to 1m
1602 // iterations if first run took < 1ns.
1603 let ns_target_total = 1_000_000; // 1ms
1604 let mut n = ns_target_total / cmp::max(1, ns_single);
1606 // if the first run took more than 1ms we don't want to just
1607 // be left doing 0 iterations on every loop. The unfortunate
1608 // side effect of not being able to do as many runs is
1609 // automatically handled by the statistical analysis below
1610 // (i.e. larger error bars).
1613 let mut total_run = Duration::new(0, 0);
1614 let samples: &mut [f64] = &mut [0.0_f64; 50];
1616 let loop_start = Instant::now();
1618 for p in &mut *samples {
1619 *p = ns_iter_inner(inner, n) as f64 / n as f64;
1622 stats::winsorize(samples, 5.0);
1623 let summ = stats::Summary::new(samples);
1625 for p in &mut *samples {
1626 let ns = ns_iter_inner(inner, 5 * n);
1627 *p = ns as f64 / (5 * n) as f64;
1630 stats::winsorize(samples, 5.0);
1631 let summ5 = stats::Summary::new(samples);
1633 let loop_run = loop_start.elapsed();
1635 // If we've run for 100ms and seem to have converged to a
1637 if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
1638 summ.median - summ5.median < summ5.median_abs_dev
1643 total_run = total_run + loop_run;
1644 // Longest we ever run for is 3s.
1645 if total_run > Duration::from_secs(3) {
1649 // If we overflow here just return the results so far. We check a
1650 // multiplier of 10 because we're about to multiply by 2 and the
1651 // next iteration of the loop will also multiply by 5 (to calculate
1652 // the summ5 result)
1653 n = match n.checked_mul(10) {
1663 use std::panic::{catch_unwind, AssertUnwindSafe};
1666 use std::sync::{Arc, Mutex};
1668 use super::{Bencher, BenchSamples, BenchMode, Sink, MonitorMsg, TestDesc, Sender, TestResult};
1670 pub fn benchmark<F>(desc: TestDesc, monitor_ch: Sender<MonitorMsg>, nocapture: bool, f: F)
1672 F: FnMut(&mut Bencher),
1674 let mut bs = Bencher {
1675 mode: BenchMode::Auto,
1680 let data = Arc::new(Mutex::new(Vec::new()));
1681 let data2 = data.clone();
1683 let oldio = if !nocapture {
1685 io::set_print(Some(Box::new(Sink(data2.clone())))),
1686 io::set_panic(Some(Box::new(Sink(data2)))),
1692 let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f)));
1694 if let Some((printio, panicio)) = oldio {
1695 io::set_print(printio);
1696 io::set_panic(panicio);
1699 let test_result = match result { //bs.bench(f) {
1700 Ok(Some(ns_iter_summ)) => {
1701 let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
1702 let mb_s = bs.bytes * 1000 / ns_iter;
1704 let bs = BenchSamples {
1706 mb_s: mb_s as usize,
1708 TestResult::TrBench(bs)
1711 // iter not called, so no data.
1712 // FIXME: error in this case?
1713 let samples: &mut [f64] = &mut [0.0_f64; 1];
1714 let bs = BenchSamples {
1715 ns_iter_summ: stats::Summary::new(samples),
1718 TestResult::TrBench(bs)
1721 TestResult::TrFailed
1725 let stdout = data.lock().unwrap().to_vec();
1726 monitor_ch.send((desc, test_result, stdout)).unwrap();
1729 pub fn run_once<F>(f: F)
1731 F: FnMut(&mut Bencher),
1733 let mut bs = Bencher {
1734 mode: BenchMode::Single,
1744 use test::{TrFailed, TrFailedMsg, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc,
1745 TestDescAndFn, TestOpts, run_test, MetricMap, StaticTestName, DynTestName,
1746 DynTestFn, ShouldPanic};
1747 use std::sync::mpsc::channel;
1752 pub fn do_not_run_ignored_tests() {
1756 let desc = TestDescAndFn {
1758 name: StaticTestName("whatever"),
1760 should_panic: ShouldPanic::No,
1763 testfn: DynTestFn(Box::new(f)),
1765 let (tx, rx) = channel();
1766 run_test(&TestOpts::new(), false, desc, tx);
1767 let (_, res, _) = rx.recv().unwrap();
1768 assert!(res != TrOk);
1772 pub fn ignored_tests_result_in_ignored() {
1774 let desc = TestDescAndFn {
1776 name: StaticTestName("whatever"),
1778 should_panic: ShouldPanic::No,
1781 testfn: DynTestFn(Box::new(f)),
1783 let (tx, rx) = channel();
1784 run_test(&TestOpts::new(), false, desc, tx);
1785 let (_, res, _) = rx.recv().unwrap();
1786 assert!(res == TrIgnored);
1790 fn test_should_panic() {
1794 let desc = TestDescAndFn {
1796 name: StaticTestName("whatever"),
1798 should_panic: ShouldPanic::Yes,
1801 testfn: DynTestFn(Box::new(f)),
1803 let (tx, rx) = channel();
1804 run_test(&TestOpts::new(), false, desc, tx);
1805 let (_, res, _) = rx.recv().unwrap();
1806 assert!(res == TrOk);
1810 fn test_should_panic_good_message() {
1812 panic!("an error message");
1814 let desc = TestDescAndFn {
1816 name: StaticTestName("whatever"),
1818 should_panic: ShouldPanic::YesWithMessage("error message"),
1821 testfn: DynTestFn(Box::new(f)),
1823 let (tx, rx) = channel();
1824 run_test(&TestOpts::new(), false, desc, tx);
1825 let (_, res, _) = rx.recv().unwrap();
1826 assert!(res == TrOk);
1830 fn test_should_panic_bad_message() {
1832 panic!("an error message");
1834 let expected = "foobar";
1835 let failed_msg = "Panic did not include expected string";
1836 let desc = TestDescAndFn {
1838 name: StaticTestName("whatever"),
1840 should_panic: ShouldPanic::YesWithMessage(expected),
1843 testfn: DynTestFn(Box::new(f)),
1845 let (tx, rx) = channel();
1846 run_test(&TestOpts::new(), false, desc, tx);
1847 let (_, res, _) = rx.recv().unwrap();
1848 assert!(res == TrFailedMsg(format!("{} '{}'", failed_msg, expected)));
1852 fn test_should_panic_but_succeeds() {
1854 let desc = TestDescAndFn {
1856 name: StaticTestName("whatever"),
1858 should_panic: ShouldPanic::Yes,
1861 testfn: DynTestFn(Box::new(f)),
1863 let (tx, rx) = channel();
1864 run_test(&TestOpts::new(), false, desc, tx);
1865 let (_, res, _) = rx.recv().unwrap();
1866 assert!(res == TrFailed);
1870 fn parse_ignored_flag() {
1872 "progname".to_string(),
1873 "filter".to_string(),
1874 "--ignored".to_string(),
1876 let opts = match parse_opts(&args) {
1878 _ => panic!("Malformed arg in parse_ignored_flag"),
1880 assert!((opts.run_ignored));
1884 pub fn filter_for_ignored_option() {
1885 // When we run ignored tests the test filter should filter out all the
1886 // unignored tests and flip the ignore flag on the rest to false
1888 let mut opts = TestOpts::new();
1889 opts.run_tests = true;
1890 opts.run_ignored = true;
1893 vec![TestDescAndFn {
1895 name: StaticTestName("1"),
1897 should_panic: ShouldPanic::No,
1900 testfn: DynTestFn(Box::new(move || {})),
1904 name: StaticTestName("2"),
1906 should_panic: ShouldPanic::No,
1909 testfn: DynTestFn(Box::new(move || {})),
1911 let filtered = filter_tests(&opts, tests);
1913 assert_eq!(filtered.len(), 1);
1914 assert_eq!(filtered[0].desc.name.to_string(), "1");
1915 assert!(!filtered[0].desc.ignore);
1919 pub fn exact_filter_match() {
1920 fn tests() -> Vec<TestDescAndFn> {
1921 vec!["base", "base::test", "base::test1", "base::test2"]
1926 name: StaticTestName(name),
1928 should_panic: ShouldPanic::No,
1931 testfn: DynTestFn(Box::new(move || {}))
1936 let substr = filter_tests(
1938 filter: Some("base".into()),
1943 assert_eq!(substr.len(), 4);
1945 let substr = filter_tests(
1947 filter: Some("bas".into()),
1952 assert_eq!(substr.len(), 4);
1954 let substr = filter_tests(
1956 filter: Some("::test".into()),
1961 assert_eq!(substr.len(), 3);
1963 let substr = filter_tests(
1965 filter: Some("base::test".into()),
1970 assert_eq!(substr.len(), 3);
1972 let exact = filter_tests(
1974 filter: Some("base".into()),
1980 assert_eq!(exact.len(), 1);
1982 let exact = filter_tests(
1984 filter: Some("bas".into()),
1990 assert_eq!(exact.len(), 0);
1992 let exact = filter_tests(
1994 filter: Some("::test".into()),
2000 assert_eq!(exact.len(), 0);
2002 let exact = filter_tests(
2004 filter: Some("base::test".into()),
2010 assert_eq!(exact.len(), 1);
2014 pub fn sort_tests() {
2015 let mut opts = TestOpts::new();
2016 opts.run_tests = true;
2019 "sha1::test".to_string(),
2020 "isize::test_to_str".to_string(),
2021 "isize::test_pow".to_string(),
2022 "test::do_not_run_ignored_tests".to_string(),
2023 "test::ignored_tests_result_in_ignored".to_string(),
2024 "test::first_free_arg_should_be_a_filter".to_string(),
2025 "test::parse_ignored_flag".to_string(),
2026 "test::filter_for_ignored_option".to_string(),
2027 "test::sort_tests".to_string(),
2031 let mut tests = Vec::new();
2032 for name in &names {
2033 let test = TestDescAndFn {
2035 name: DynTestName((*name).clone()),
2037 should_panic: ShouldPanic::No,
2040 testfn: DynTestFn(Box::new(testfn)),
2046 let filtered = filter_tests(&opts, tests);
2048 let expected = vec![
2049 "isize::test_pow".to_string(),
2050 "isize::test_to_str".to_string(),
2051 "sha1::test".to_string(),
2052 "test::do_not_run_ignored_tests".to_string(),
2053 "test::filter_for_ignored_option".to_string(),
2054 "test::first_free_arg_should_be_a_filter".to_string(),
2055 "test::ignored_tests_result_in_ignored".to_string(),
2056 "test::parse_ignored_flag".to_string(),
2057 "test::sort_tests".to_string(),
2060 for (a, b) in expected.iter().zip(filtered) {
2061 assert!(*a == b.desc.name.to_string());
2066 pub fn test_metricmap_compare() {
2067 let mut m1 = MetricMap::new();
2068 let mut m2 = MetricMap::new();
2069 m1.insert_metric("in-both-noise", 1000.0, 200.0);
2070 m2.insert_metric("in-both-noise", 1100.0, 200.0);
2072 m1.insert_metric("in-first-noise", 1000.0, 2.0);
2073 m2.insert_metric("in-second-noise", 1000.0, 2.0);
2075 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
2076 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
2078 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
2079 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
2081 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
2082 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
2084 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
2085 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
2089 pub fn test_bench_once_no_iter() {
2090 fn f(_: &mut Bencher) {}
2095 pub fn test_bench_once_iter() {
2096 fn f(b: &mut Bencher) {
2103 pub fn test_bench_no_iter() {
2104 fn f(_: &mut Bencher) {}
2106 let (tx, rx) = channel();
2108 let desc = TestDesc {
2109 name: StaticTestName("f"),
2111 should_panic: ShouldPanic::No,
2115 ::bench::benchmark(desc,
2123 pub fn test_bench_iter() {
2124 fn f(b: &mut Bencher) {
2128 let (tx, rx) = channel();
2130 let desc = TestDesc {
2131 name: StaticTestName("f"),
2133 should_panic: ShouldPanic::No,
2137 ::bench::benchmark(desc,