1 //! Benchmarking module.
2 pub use std::hint::black_box;
5 event::CompletedTest, helpers::sink::Sink, options::BenchMode, test_result::TestResult,
6 types::TestDesc, Sender,
12 use std::panic::{catch_unwind, AssertUnwindSafe};
13 use std::sync::{Arc, Mutex};
14 use std::time::{Duration, Instant};
16 /// Manager of the benchmarking runs.
18 /// This is fed into functions marked with `#[bench]` to allow for
19 /// set-up & tear-down before running a piece of code repeatedly via a
24 summary: Option<stats::Summary>,
29 /// Callback for benchmark functions to run in their body.
30 pub fn iter<T, F>(&mut self, mut inner: F)
34 if self.mode == BenchMode::Single {
35 ns_iter_inner(&mut inner, 1);
39 self.summary = Some(iter(&mut inner));
42 pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary>
44 F: FnMut(&mut Bencher),
51 #[derive(Debug, Clone, PartialEq)]
52 pub struct BenchSamples {
53 pub ns_iter_summ: stats::Summary,
57 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
59 let mut output = String::new();
61 let median = bs.ns_iter_summ.median as usize;
62 let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;
65 .write_fmt(format_args!(
66 "{:>11} ns/iter (+/- {})",
67 fmt_thousands_sep(median, ','),
68 fmt_thousands_sep(deviation, ',')
72 output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
77 // Format a number with thousands separators
78 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
80 let mut output = String::new();
81 let mut trailing = false;
82 for &pow in &[9, 6, 3, 0] {
83 let base = 10_usize.pow(pow);
84 if pow == 0 || trailing || n / base != 0 {
86 output.write_fmt(format_args!("{}", n / base)).unwrap();
88 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
101 fn ns_from_dur(dur: Duration) -> u64 {
102 dur.as_secs() * 1_000_000_000 + (dur.subsec_nanos() as u64)
105 fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64
109 let start = Instant::now();
113 ns_from_dur(start.elapsed())
116 pub fn iter<T, F>(inner: &mut F) -> stats::Summary
120 // Initial bench run to get ballpark figure.
121 let ns_single = ns_iter_inner(inner, 1);
123 // Try to estimate iter count for 1ms falling back to 1m
124 // iterations if first run took < 1ns.
125 let ns_target_total = 1_000_000; // 1ms
126 let mut n = ns_target_total / cmp::max(1, ns_single);
128 // if the first run took more than 1ms we don't want to just
129 // be left doing 0 iterations on every loop. The unfortunate
130 // side effect of not being able to do as many runs is
131 // automatically handled by the statistical analysis below
132 // (i.e., larger error bars).
135 let mut total_run = Duration::new(0, 0);
136 let samples: &mut [f64] = &mut [0.0_f64; 50];
138 let loop_start = Instant::now();
140 for p in &mut *samples {
141 *p = ns_iter_inner(inner, n) as f64 / n as f64;
144 stats::winsorize(samples, 5.0);
145 let summ = stats::Summary::new(samples);
147 for p in &mut *samples {
148 let ns = ns_iter_inner(inner, 5 * n);
149 *p = ns as f64 / (5 * n) as f64;
152 stats::winsorize(samples, 5.0);
153 let summ5 = stats::Summary::new(samples);
155 let loop_run = loop_start.elapsed();
157 // If we've run for 100ms and seem to have converged to a
159 if loop_run > Duration::from_millis(100)
160 && summ.median_abs_dev_pct < 1.0
161 && summ.median - summ5.median < summ5.median_abs_dev
166 total_run = total_run + loop_run;
167 // Longest we ever run for is 3s.
168 if total_run > Duration::from_secs(3) {
172 // If we overflow here just return the results so far. We check a
173 // multiplier of 10 because we're about to multiply by 2 and the
174 // next iteration of the loop will also multiply by 5 (to calculate
176 n = match n.checked_mul(10) {
185 pub fn benchmark<F>(desc: TestDesc, monitor_ch: Sender<CompletedTest>, nocapture: bool, f: F)
187 F: FnMut(&mut Bencher),
189 let mut bs = Bencher { mode: BenchMode::Auto, summary: None, bytes: 0 };
191 let data = Arc::new(Mutex::new(Vec::new()));
192 let oldio = if !nocapture {
194 io::set_print(Some(Sink::new_boxed(&data))),
195 io::set_panic(Some(Sink::new_boxed(&data))),
201 let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f)));
203 if let Some((printio, panicio)) = oldio {
204 io::set_print(printio);
205 io::set_panic(panicio);
208 let test_result = match result {
210 Ok(Some(ns_iter_summ)) => {
211 let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
212 let mb_s = bs.bytes * 1000 / ns_iter;
214 let bs = BenchSamples { ns_iter_summ, mb_s: mb_s as usize };
215 TestResult::TrBench(bs)
218 // iter not called, so no data.
219 // FIXME: error in this case?
220 let samples: &mut [f64] = &mut [0.0_f64; 1];
221 let bs = BenchSamples { ns_iter_summ: stats::Summary::new(samples), mb_s: 0 };
222 TestResult::TrBench(bs)
224 Err(_) => TestResult::TrFailed,
227 let stdout = data.lock().unwrap().to_vec();
228 let message = CompletedTest::new(desc, test_result, None, stdout);
229 monitor_ch.send(message).unwrap();
232 pub fn run_once<F>(f: F)
234 F: FnMut(&mut Bencher),
236 let mut bs = Bencher { mode: BenchMode::Single, summary: None, bytes: 0 };