library/test/src/console.rs

   1 //! Module providing interface for running tests in the console.
   2
   3 use std::fs::File;
   4 use std::io;
   5 use std::io::prelude::Write;
   6 use std::time::Instant;
   7
   8 use super::{
   9     bench::fmt_bench_samples,
  10     cli::TestOpts,
  11     event::{CompletedTest, TestEvent},
  12     filter_tests,
  13     formatters::{JsonFormatter, JunitFormatter, OutputFormatter, PrettyFormatter, TerseFormatter},
  14     helpers::{concurrency::get_concurrency, metrics::MetricMap},
  15     options::{Options, OutputFormat},
  16     run_tests, term,
  17     test_result::TestResult,
  18     time::{TestExecTime, TestSuiteExecTime},
  19     types::{NamePadding, TestDesc, TestDescAndFn},
  20 };
  21
  22 /// Generic wrapper over stdout.
  23 pub enum OutputLocation<T> {
  24     Pretty(Box<term::StdoutTerminal>),
  25     Raw(T),
  26 }
  27
  28 impl<T: Write> Write for OutputLocation<T> {
  29     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
  30         match *self {
  31             OutputLocation::Pretty(ref mut term) => term.write(buf),
  32             OutputLocation::Raw(ref mut stdout) => stdout.write(buf),
  33         }
  34     }
  35
  36     fn flush(&mut self) -> io::Result<()> {
  37         match *self {
  38             OutputLocation::Pretty(ref mut term) => term.flush(),
  39             OutputLocation::Raw(ref mut stdout) => stdout.flush(),
  40         }
  41     }
  42 }
  43
  44 pub struct ConsoleTestState {
  45     pub log_out: Option<File>,
  46     pub total: usize,
  47     pub passed: usize,
  48     pub failed: usize,
  49     pub ignored: usize,
  50     pub filtered_out: usize,
  51     pub measured: usize,
  52     pub exec_time: Option<TestSuiteExecTime>,
  53     pub metrics: MetricMap,
  54     pub failures: Vec<(TestDesc, Vec<u8>)>,
  55     pub not_failures: Vec<(TestDesc, Vec<u8>)>,
  56     pub time_failures: Vec<(TestDesc, Vec<u8>)>,
  57     pub options: Options,
  58 }
  59
  60 impl ConsoleTestState {
  61     pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState> {
  62         let log_out = match opts.logfile {
  63             Some(ref path) => Some(File::create(path)?),
  64             None => None,
  65         };
  66
  67         Ok(ConsoleTestState {
  68             log_out,
  69             total: 0,
  70             passed: 0,
  71             failed: 0,
  72             ignored: 0,
  73             filtered_out: 0,
  74             measured: 0,
  75             exec_time: None,
  76             metrics: MetricMap::new(),
  77             failures: Vec::new(),
  78             not_failures: Vec::new(),
  79             time_failures: Vec::new(),
  80             options: opts.options,
  81         })
  82     }
  83
  84     pub fn write_log<F, S>(&mut self, msg: F) -> io::Result<()>
  85     where
  86         S: AsRef<str>,
  87         F: FnOnce() -> S,
  88     {
  89         match self.log_out {
  90             None => Ok(()),
  91             Some(ref mut o) => {
  92                 let msg = msg();
  93                 let msg = msg.as_ref();
  94                 o.write_all(msg.as_bytes())
  95             }
  96         }
  97     }
  98
  99     pub fn write_log_result(
 100         &mut self,
 101         test: &TestDesc,
 102         result: &TestResult,
 103         exec_time: Option<&TestExecTime>,
 104     ) -> io::Result<()> {
 105         self.write_log(|| {
 106             let TestDesc {
 107                 name,
 108                 #[cfg(not(bootstrap))]
 109                 ignore_message,
 110                 ..
 111             } = test;
 112             format!(
 113                 "{} {}",
 114                 match *result {
 115                     TestResult::TrOk => "ok".to_owned(),
 116                     TestResult::TrFailed => "failed".to_owned(),
 117                     TestResult::TrFailedMsg(ref msg) => format!("failed: {}", msg),
 118                     TestResult::TrIgnored => {
 119                         #[cfg(not(bootstrap))]
 120                         if let Some(msg) = ignore_message {
 121                             format!("ignored, {}", msg)
 122                         } else {
 123                             "ignored".to_owned()
 124                         }
 125                         #[cfg(bootstrap)]
 126                         "ignored".to_owned()
 127                     }
 128                     TestResult::TrBench(ref bs) => fmt_bench_samples(bs),
 129                     TestResult::TrTimedFail => "failed (time limit exceeded)".to_owned(),
 130                 },
 131                 name,
 132             )
 133         })?;
 134         if let Some(exec_time) = exec_time {
 135             self.write_log(|| format!(" <{}>", exec_time))?;
 136         }
 137         self.write_log(|| "\n")
 138     }
 139
 140     fn current_test_count(&self) -> usize {
 141         self.passed + self.failed + self.ignored + self.measured
 142     }
 143 }
 144
 145 // List the tests to console, and optionally to logfile. Filters are honored.
 146 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
 147     let mut output = match term::stdout() {
 148         None => OutputLocation::Raw(io::stdout()),
 149         Some(t) => OutputLocation::Pretty(t),
 150     };
 151
 152     let quiet = opts.format == OutputFormat::Terse;
 153     let mut st = ConsoleTestState::new(opts)?;
 154
 155     let mut ntest = 0;
 156     let mut nbench = 0;
 157
 158     for test in filter_tests(&opts, tests) {
 159         use crate::TestFn::*;
 160
 161         let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test;
 162
 163         let fntype = match testfn {
 164             StaticTestFn(..) | DynTestFn(..) => {
 165                 ntest += 1;
 166                 "test"
 167             }
 168             StaticBenchFn(..) | DynBenchFn(..) => {
 169                 nbench += 1;
 170                 "benchmark"
 171             }
 172         };
 173
 174         writeln!(output, "{}: {}", name, fntype)?;
 175         st.write_log(|| format!("{} {}\n", fntype, name))?;
 176     }
 177
 178     fn plural(count: u32, s: &str) -> String {
 179         match count {
 180             1 => format!("{} {}", 1, s),
 181             n => format!("{} {}s", n, s),
 182         }
 183     }
 184
 185     if !quiet {
 186         if ntest != 0 || nbench != 0 {
 187             writeln!(output)?;
 188         }
 189
 190         writeln!(output, "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark"))?;
 191     }
 192
 193     Ok(())
 194 }
 195
 196 // Updates `ConsoleTestState` depending on result of the test execution.
 197 fn handle_test_result(st: &mut ConsoleTestState, completed_test: CompletedTest) {
 198     let test = completed_test.desc;
 199     let stdout = completed_test.stdout;
 200     match completed_test.result {
 201         TestResult::TrOk => {
 202             st.passed += 1;
 203             st.not_failures.push((test, stdout));
 204         }
 205         TestResult::TrIgnored => st.ignored += 1,
 206         TestResult::TrBench(bs) => {
 207             st.metrics.insert_metric(
 208                 test.name.as_slice(),
 209                 bs.ns_iter_summ.median,
 210                 bs.ns_iter_summ.max - bs.ns_iter_summ.min,
 211             );
 212             st.measured += 1
 213         }
 214         TestResult::TrFailed => {
 215             st.failed += 1;
 216             st.failures.push((test, stdout));
 217         }
 218         TestResult::TrFailedMsg(msg) => {
 219             st.failed += 1;
 220             let mut stdout = stdout;
 221             stdout.extend_from_slice(format!("note: {}", msg).as_bytes());
 222             st.failures.push((test, stdout));
 223         }
 224         TestResult::TrTimedFail => {
 225             st.failed += 1;
 226             st.time_failures.push((test, stdout));
 227         }
 228     }
 229 }
 230
 231 // Handler for events that occur during test execution.
 232 // It is provided as a callback to the `run_tests` function.
 233 fn on_test_event(
 234     event: &TestEvent,
 235     st: &mut ConsoleTestState,
 236     out: &mut dyn OutputFormatter,
 237 ) -> io::Result<()> {
 238     match (*event).clone() {
 239         TestEvent::TeFiltered(ref filtered_tests, shuffle_seed) => {
 240             st.total = filtered_tests.len();
 241             out.write_run_start(filtered_tests.len(), shuffle_seed)?;
 242         }
 243         TestEvent::TeFilteredOut(filtered_out) => {
 244             st.filtered_out = filtered_out;
 245         }
 246         TestEvent::TeWait(ref test) => out.write_test_start(test)?,
 247         TestEvent::TeTimeout(ref test) => out.write_timeout(test)?,
 248         TestEvent::TeResult(completed_test) => {
 249             let test = &completed_test.desc;
 250             let result = &completed_test.result;
 251             let exec_time = &completed_test.exec_time;
 252             let stdout = &completed_test.stdout;
 253
 254             st.write_log_result(test, result, exec_time.as_ref())?;
 255             out.write_result(test, result, exec_time.as_ref(), &*stdout, st)?;
 256             handle_test_result(st, completed_test);
 257         }
 258     }
 259
 260     Ok(())
 261 }
 262
 263 /// A simple console test runner.
 264 /// Runs provided tests reporting process and results to the stdout.
 265 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
 266     let output = match term::stdout() {
 267         None => OutputLocation::Raw(io::stdout()),
 268         Some(t) => OutputLocation::Pretty(t),
 269     };
 270
 271     let max_name_len = tests
 272         .iter()
 273         .max_by_key(|t| len_if_padded(*t))
 274         .map(|t| t.desc.name.as_slice().len())
 275         .unwrap_or(0);
 276
 277     let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1;
 278
 279     let mut out: Box<dyn OutputFormatter> = match opts.format {
 280         OutputFormat::Pretty => Box::new(PrettyFormatter::new(
 281             output,
 282             opts.use_color(),
 283             max_name_len,
 284             is_multithreaded,
 285             opts.time_options,
 286         )),
 287         OutputFormat::Terse => {
 288             Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded))
 289         }
 290         OutputFormat::Json => Box::new(JsonFormatter::new(output)),
 291         OutputFormat::Junit => Box::new(JunitFormatter::new(output)),
 292     };
 293     let mut st = ConsoleTestState::new(opts)?;
 294
 295     // Prevent the usage of `Instant` in some cases:
 296     // - It's currently not supported for wasm targets.
 297     // - We disable it for miri because it's not available when isolation is enabled.
 298     let is_instant_supported = !cfg!(target_family = "wasm") && !cfg!(miri);
 299
 300     let start_time = is_instant_supported.then(Instant::now);
 301     run_tests(opts, tests, |x| on_test_event(&x, &mut st, &mut *out))?;
 302     st.exec_time = start_time.map(|t| TestSuiteExecTime(t.elapsed()));
 303
 304     assert!(st.current_test_count() == st.total);
 305
 306     out.write_run_finish(&st)
 307 }
 308
 309 // Calculates padding for given test description.
 310 fn len_if_padded(t: &TestDescAndFn) -> usize {
 311     match t.testfn.padding() {
 312         NamePadding::PadNone => 0,
 313         NamePadding::PadOnRight => t.desc.name.as_slice().len(),
 314     }
 315 }