library/test/src/console.rs

   1 //! Module providing interface for running tests in the console.
   2
   3 use std::fs::File;
   4 use std::io;
   5 use std::io::prelude::Write;
   6 use std::time::Instant;
   7
   8 use super::{
   9     bench::fmt_bench_samples,
  10     cli::TestOpts,
  11     event::{CompletedTest, TestEvent},
  12     filter_tests,
  13     formatters::{JsonFormatter, JunitFormatter, OutputFormatter, PrettyFormatter, TerseFormatter},
  14     helpers::{concurrency::get_concurrency, metrics::MetricMap},
  15     options::{Options, OutputFormat},
  16     run_tests, term,
  17     test_result::TestResult,
  18     time::{TestExecTime, TestSuiteExecTime},
  19     types::{NamePadding, TestDesc, TestDescAndFn},
  20 };
  21
  22 /// Generic wrapper over stdout.
  23 pub enum OutputLocation<T> {
  24     Pretty(Box<term::StdoutTerminal>),
  25     Raw(T),
  26 }
  27
  28 impl<T: Write> Write for OutputLocation<T> {
  29     fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
  30         match *self {
  31             OutputLocation::Pretty(ref mut term) => term.write(buf),
  32             OutputLocation::Raw(ref mut stdout) => stdout.write(buf),
  33         }
  34     }
  35
  36     fn flush(&mut self) -> io::Result<()> {
  37         match *self {
  38             OutputLocation::Pretty(ref mut term) => term.flush(),
  39             OutputLocation::Raw(ref mut stdout) => stdout.flush(),
  40         }
  41     }
  42 }
  43
  44 pub struct ConsoleTestState {
  45     pub log_out: Option<File>,
  46     pub total: usize,
  47     pub passed: usize,
  48     pub failed: usize,
  49     pub ignored: usize,
  50     pub filtered_out: usize,
  51     pub measured: usize,
  52     pub exec_time: Option<TestSuiteExecTime>,
  53     pub metrics: MetricMap,
  54     pub failures: Vec<(TestDesc, Vec<u8>)>,
  55     pub not_failures: Vec<(TestDesc, Vec<u8>)>,
  56     pub ignores: Vec<(TestDesc, Vec<u8>)>,
  57     pub time_failures: Vec<(TestDesc, Vec<u8>)>,
  58     pub options: Options,
  59 }
  60
  61 impl ConsoleTestState {
  62     pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState> {
  63         let log_out = match opts.logfile {
  64             Some(ref path) => Some(File::create(path)?),
  65             None => None,
  66         };
  67
  68         Ok(ConsoleTestState {
  69             log_out,
  70             total: 0,
  71             passed: 0,
  72             failed: 0,
  73             ignored: 0,
  74             filtered_out: 0,
  75             measured: 0,
  76             exec_time: None,
  77             metrics: MetricMap::new(),
  78             failures: Vec::new(),
  79             not_failures: Vec::new(),
  80             ignores: Vec::new(),
  81             time_failures: Vec::new(),
  82             options: opts.options,
  83         })
  84     }
  85
  86     pub fn write_log<F, S>(&mut self, msg: F) -> io::Result<()>
  87     where
  88         S: AsRef<str>,
  89         F: FnOnce() -> S,
  90     {
  91         match self.log_out {
  92             None => Ok(()),
  93             Some(ref mut o) => {
  94                 let msg = msg();
  95                 let msg = msg.as_ref();
  96                 o.write_all(msg.as_bytes())
  97             }
  98         }
  99     }
 100
 101     pub fn write_log_result(
 102         &mut self,
 103         test: &TestDesc,
 104         result: &TestResult,
 105         exec_time: Option<&TestExecTime>,
 106     ) -> io::Result<()> {
 107         self.write_log(|| {
 108             let TestDesc { name, ignore_message, .. } = test;
 109             format!(
 110                 "{} {}",
 111                 match *result {
 112                     TestResult::TrOk => "ok".to_owned(),
 113                     TestResult::TrFailed => "failed".to_owned(),
 114                     TestResult::TrFailedMsg(ref msg) => format!("failed: {msg}"),
 115                     TestResult::TrIgnored => {
 116                         if let Some(msg) = ignore_message {
 117                             format!("ignored: {msg}")
 118                         } else {
 119                             "ignored".to_owned()
 120                         }
 121                     }
 122                     TestResult::TrBench(ref bs) => fmt_bench_samples(bs),
 123                     TestResult::TrTimedFail => "failed (time limit exceeded)".to_owned(),
 124                 },
 125                 name,
 126             )
 127         })?;
 128         if let Some(exec_time) = exec_time {
 129             self.write_log(|| format!(" <{exec_time}>"))?;
 130         }
 131         self.write_log(|| "\n")
 132     }
 133
 134     fn current_test_count(&self) -> usize {
 135         self.passed + self.failed + self.ignored + self.measured
 136     }
 137 }
 138
 139 // List the tests to console, and optionally to logfile. Filters are honored.
 140 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
 141     let mut output = match term::stdout() {
 142         None => OutputLocation::Raw(io::stdout().lock()),
 143         Some(t) => OutputLocation::Pretty(t),
 144     };
 145
 146     let quiet = opts.format == OutputFormat::Terse;
 147     let mut st = ConsoleTestState::new(opts)?;
 148
 149     let mut ntest = 0;
 150     let mut nbench = 0;
 151
 152     for test in filter_tests(opts, tests).into_iter() {
 153         use crate::TestFn::*;
 154
 155         let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test;
 156
 157         let fntype = match testfn {
 158             StaticTestFn(..) | DynTestFn(..) => {
 159                 ntest += 1;
 160                 "test"
 161             }
 162             StaticBenchFn(..) | DynBenchFn(..) => {
 163                 nbench += 1;
 164                 "benchmark"
 165             }
 166         };
 167
 168         writeln!(output, "{name}: {fntype}")?;
 169         st.write_log(|| format!("{fntype} {name}\n"))?;
 170     }
 171
 172     fn plural(count: u32, s: &str) -> String {
 173         match count {
 174             1 => format!("1 {s}"),
 175             n => format!("{n} {s}s"),
 176         }
 177     }
 178
 179     if !quiet {
 180         if ntest != 0 || nbench != 0 {
 181             writeln!(output)?;
 182         }
 183
 184         writeln!(output, "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark"))?;
 185     }
 186
 187     Ok(())
 188 }
 189
 190 // Updates `ConsoleTestState` depending on result of the test execution.
 191 fn handle_test_result(st: &mut ConsoleTestState, completed_test: CompletedTest) {
 192     let test = completed_test.desc;
 193     let stdout = completed_test.stdout;
 194     match completed_test.result {
 195         TestResult::TrOk => {
 196             st.passed += 1;
 197             st.not_failures.push((test, stdout));
 198         }
 199         TestResult::TrIgnored => {
 200             st.ignored += 1;
 201             st.ignores.push((test, stdout));
 202         }
 203         TestResult::TrBench(bs) => {
 204             st.metrics.insert_metric(
 205                 test.name.as_slice(),
 206                 bs.ns_iter_summ.median,
 207                 bs.ns_iter_summ.max - bs.ns_iter_summ.min,
 208             );
 209             st.measured += 1
 210         }
 211         TestResult::TrFailed => {
 212             st.failed += 1;
 213             st.failures.push((test, stdout));
 214         }
 215         TestResult::TrFailedMsg(msg) => {
 216             st.failed += 1;
 217             let mut stdout = stdout;
 218             stdout.extend_from_slice(format!("note: {msg}").as_bytes());
 219             st.failures.push((test, stdout));
 220         }
 221         TestResult::TrTimedFail => {
 222             st.failed += 1;
 223             st.time_failures.push((test, stdout));
 224         }
 225     }
 226 }
 227
 228 // Handler for events that occur during test execution.
 229 // It is provided as a callback to the `run_tests` function.
 230 fn on_test_event(
 231     event: &TestEvent,
 232     st: &mut ConsoleTestState,
 233     out: &mut dyn OutputFormatter,
 234 ) -> io::Result<()> {
 235     match (*event).clone() {
 236         TestEvent::TeFiltered(filtered_tests, shuffle_seed) => {
 237             st.total = filtered_tests;
 238             out.write_run_start(filtered_tests, shuffle_seed)?;
 239         }
 240         TestEvent::TeFilteredOut(filtered_out) => {
 241             st.filtered_out = filtered_out;
 242         }
 243         TestEvent::TeWait(ref test) => out.write_test_start(test)?,
 244         TestEvent::TeTimeout(ref test) => out.write_timeout(test)?,
 245         TestEvent::TeResult(completed_test) => {
 246             let test = &completed_test.desc;
 247             let result = &completed_test.result;
 248             let exec_time = &completed_test.exec_time;
 249             let stdout = &completed_test.stdout;
 250
 251             st.write_log_result(test, result, exec_time.as_ref())?;
 252             out.write_result(test, result, exec_time.as_ref(), stdout, st)?;
 253             handle_test_result(st, completed_test);
 254         }
 255     }
 256
 257     Ok(())
 258 }
 259
 260 /// A simple console test runner.
 261 /// Runs provided tests reporting process and results to the stdout.
 262 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
 263     let output = match term::stdout() {
 264         None => OutputLocation::Raw(io::stdout()),
 265         Some(t) => OutputLocation::Pretty(t),
 266     };
 267
 268     let max_name_len = tests
 269         .iter()
 270         .max_by_key(|t| len_if_padded(t))
 271         .map(|t| t.desc.name.as_slice().len())
 272         .unwrap_or(0);
 273
 274     let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1;
 275
 276     let mut out: Box<dyn OutputFormatter> = match opts.format {
 277         OutputFormat::Pretty => Box::new(PrettyFormatter::new(
 278             output,
 279             opts.use_color(),
 280             max_name_len,
 281             is_multithreaded,
 282             opts.time_options,
 283         )),
 284         OutputFormat::Terse => {
 285             Box::new(TerseFormatter::new(output, opts.use_color(), max_name_len, is_multithreaded))
 286         }
 287         OutputFormat::Json => Box::new(JsonFormatter::new(output)),
 288         OutputFormat::Junit => Box::new(JunitFormatter::new(output)),
 289     };
 290     let mut st = ConsoleTestState::new(opts)?;
 291
 292     // Prevent the usage of `Instant` in some cases:
 293     // - It's currently not supported for wasm targets.
 294     // - We disable it for miri because it's not available when isolation is enabled.
 295     let is_instant_supported = !cfg!(target_family = "wasm") && !cfg!(miri);
 296
 297     let start_time = is_instant_supported.then(Instant::now);
 298     run_tests(opts, tests, |x| on_test_event(&x, &mut st, &mut *out))?;
 299     st.exec_time = start_time.map(|t| TestSuiteExecTime(t.elapsed()));
 300
 301     assert!(opts.fail_fast || st.current_test_count() == st.total);
 302
 303     out.write_run_finish(&st)
 304 }
 305
 306 // Calculates padding for given test description.
 307 fn len_if_padded(t: &TestDescAndFn) -> usize {
 308     match t.testfn.padding() {
 309         NamePadding::PadNone => 0,
 310         NamePadding::PadOnRight => t.desc.name.as_slice().len(),
 311     }
 312 }