1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![allow(missing_docs)]
12 #![allow(deprecated)] // Float
14 use std::cmp::Ordering::{self, Less, Greater, Equal};
17 fn local_cmp(x: f64, y: f64) -> Ordering {
18 // arbitrarily decide that NaNs are larger than everything.
21 } else if x.is_nan() {
32 fn local_sort(v: &mut [f64]) {
33 v.sort_by(|x: &f64, y: &f64| local_cmp(*x, *y));
36 /// Trait that provides simple descriptive statistics on a univariate set of numeric samples.
39 /// Sum of the samples.
41 /// Note: this method sacrifices performance at the altar of accuracy
42 /// Depends on IEEE-754 arithmetic guarantees. See proof of correctness at:
43 /// ["Adaptive Precision Floating-Point Arithmetic and Fast Robust Geometric Predicates"]
44 /// (http://www.cs.cmu.edu/~quake-papers/robust-arithmetic.ps)
47 /// Minimum value of the samples.
50 /// Maximum value of the samples.
53 /// Arithmetic mean (average) of the samples: sum divided by sample-count.
55 /// See: https://en.wikipedia.org/wiki/Arithmetic_mean
56 fn mean(&self) -> f64;
58 /// Median of the samples: value separating the lower half of the samples from the higher half.
59 /// Equal to `self.percentile(50.0)`.
61 /// See: https://en.wikipedia.org/wiki/Median
62 fn median(&self) -> f64;
64 /// Variance of the samples: bias-corrected mean of the squares of the differences of each
65 /// sample from the sample mean. Note that this calculates the _sample variance_ rather than the
66 /// population variance, which is assumed to be unknown. It therefore corrects the `(n-1)/n`
67 /// bias that would appear if we calculated a population variance, by dividing by `(n-1)` rather
70 /// See: https://en.wikipedia.org/wiki/Variance
73 /// Standard deviation: the square root of the sample variance.
75 /// Note: this is not a robust statistic for non-normal distributions. Prefer the
76 /// `median_abs_dev` for unknown distributions.
78 /// See: https://en.wikipedia.org/wiki/Standard_deviation
79 fn std_dev(&self) -> f64;
81 /// Standard deviation as a percent of the mean value. See `std_dev` and `mean`.
83 /// Note: this is not a robust statistic for non-normal distributions. Prefer the
84 /// `median_abs_dev_pct` for unknown distributions.
85 fn std_dev_pct(&self) -> f64;
87 /// Scaled median of the absolute deviations of each sample from the sample median. This is a
88 /// robust (distribution-agnostic) estimator of sample variability. Use this in preference to
89 /// `std_dev` if you cannot assume your sample is normally distributed. Note that this is scaled
90 /// by the constant `1.4826` to allow its use as a consistent estimator for the standard
93 /// See: http://en.wikipedia.org/wiki/Median_absolute_deviation
94 fn median_abs_dev(&self) -> f64;
96 /// Median absolute deviation as a percent of the median. See `median_abs_dev` and `median`.
97 fn median_abs_dev_pct(&self) -> f64;
99 /// Percentile: the value below which `pct` percent of the values in `self` fall. For example,
100 /// percentile(95.0) will return the value `v` such that 95% of the samples `s` in `self`
101 /// satisfy `s <= v`.
103 /// Calculated by linear interpolation between closest ranks.
105 /// See: http://en.wikipedia.org/wiki/Percentile
106 fn percentile(&self, pct: f64) -> f64;
108 /// Quartiles of the sample: three values that divide the sample into four equal groups, each
109 /// with 1/4 of the data. The middle value is the median. See `median` and `percentile`. This
110 /// function may calculate the 3 quartiles more efficiently than 3 calls to `percentile`, but
111 /// is otherwise equivalent.
113 /// See also: https://en.wikipedia.org/wiki/Quartile
114 fn quartiles(&self) -> (f64, f64, f64);
116 /// Inter-quartile range: the difference between the 25th percentile (1st quartile) and the 75th
117 /// percentile (3rd quartile). See `quartiles`.
119 /// See also: https://en.wikipedia.org/wiki/Interquartile_range
120 fn iqr(&self) -> f64;
123 /// Extracted collection of all the summary statistics of a sample set.
124 #[derive(Clone, PartialEq)]
125 #[allow(missing_docs)]
134 pub std_dev_pct: f64,
135 pub median_abs_dev: f64,
136 pub median_abs_dev_pct: f64,
137 pub quartiles: (f64, f64, f64),
142 /// Construct a new summary of a sample set.
143 pub fn new(samples: &[f64]) -> Summary {
148 mean: samples.mean(),
149 median: samples.median(),
151 std_dev: samples.std_dev(),
152 std_dev_pct: samples.std_dev_pct(),
153 median_abs_dev: samples.median_abs_dev(),
154 median_abs_dev_pct: samples.median_abs_dev_pct(),
155 quartiles: samples.quartiles(),
161 impl Stats for [f64] {
162 // FIXME #11059 handle NaN, inf and overflow
163 fn sum(&self) -> f64 {
164 let mut partials = vec![];
169 // This inner loop applies `hi`/`lo` summation to each
170 // partial so that the list of partial sums remains exact.
171 for i in 0..partials.len() {
172 let mut y: f64 = partials[i];
173 if x.abs() < y.abs() {
174 mem::swap(&mut x, &mut y);
176 // Rounded `x+y` is stored in `hi` with round-off stored in
177 // `lo`. Together `hi+lo` are exactly equal to `x+y`.
179 let lo = y - (hi - x);
186 if j >= partials.len() {
190 partials.truncate(j + 1);
194 partials.iter().fold(zero, |p, q| p + *q)
197 fn min(&self) -> f64 {
198 assert!(!self.is_empty());
199 self.iter().fold(self[0], |p, q| p.min(*q))
202 fn max(&self) -> f64 {
203 assert!(!self.is_empty());
204 self.iter().fold(self[0], |p, q| p.max(*q))
207 fn mean(&self) -> f64 {
208 assert!(!self.is_empty());
209 self.sum() / (self.len() as f64)
212 fn median(&self) -> f64 {
213 self.percentile(50 as f64)
216 fn var(&self) -> f64 {
220 let mean = self.mean();
221 let mut v: f64 = 0.0;
226 // NB: this is _supposed to be_ len-1, not len. If you
227 // change it back to len, you will be calculating a
228 // population variance, not a sample variance.
229 let denom = (self.len() - 1) as f64;
234 fn std_dev(&self) -> f64 {
238 fn std_dev_pct(&self) -> f64 {
239 let hundred = 100 as f64;
240 (self.std_dev() / self.mean()) * hundred
243 fn median_abs_dev(&self) -> f64 {
244 let med = self.median();
245 let abs_devs: Vec<f64> = self.iter().map(|&v| (med - v).abs()).collect();
246 // This constant is derived by smarter statistics brains than me, but it is
247 // consistent with how R and other packages treat the MAD.
249 abs_devs.median() * number
252 fn median_abs_dev_pct(&self) -> f64 {
253 let hundred = 100 as f64;
254 (self.median_abs_dev() / self.median()) * hundred
257 fn percentile(&self, pct: f64) -> f64 {
258 let mut tmp = self.to_vec();
259 local_sort(&mut tmp);
260 percentile_of_sorted(&tmp, pct)
263 fn quartiles(&self) -> (f64, f64, f64) {
264 let mut tmp = self.to_vec();
265 local_sort(&mut tmp);
267 let a = percentile_of_sorted(&tmp, first);
269 let b = percentile_of_sorted(&tmp, secound);
271 let c = percentile_of_sorted(&tmp, third);
275 fn iqr(&self) -> f64 {
276 let (a, _, c) = self.quartiles();
282 // Helper function: extract a value representing the `pct` percentile of a sorted sample-set, using
283 // linear interpolation. If samples are not sorted, return nonsensical value.
284 fn percentile_of_sorted(sorted_samples: &[f64], pct: f64) -> f64 {
285 assert!(!sorted_samples.is_empty());
286 if sorted_samples.len() == 1 {
287 return sorted_samples[0];
290 assert!(zero <= pct);
291 let hundred = 100f64;
292 assert!(pct <= hundred);
294 return sorted_samples[sorted_samples.len() - 1];
296 let length = (sorted_samples.len() - 1) as f64;
297 let rank = (pct / hundred) * length;
298 let lrank = rank.floor();
299 let d = rank - lrank;
300 let n = lrank as usize;
301 let lo = sorted_samples[n];
302 let hi = sorted_samples[n + 1];
307 /// Winsorize a set of samples, replacing values above the `100-pct` percentile
308 /// and below the `pct` percentile with those percentiles themselves. This is a
309 /// way of minimizing the effect of outliers, at the cost of biasing the sample.
310 /// It differs from trimming in that it does not change the number of samples,
311 /// just changes the values of those that are outliers.
313 /// See: http://en.wikipedia.org/wiki/Winsorising
314 pub fn winsorize(samples: &mut [f64], pct: f64) {
315 let mut tmp = samples.to_vec();
316 local_sort(&mut tmp);
317 let lo = percentile_of_sorted(&tmp, pct);
318 let hundred = 100 as f64;
319 let hi = percentile_of_sorted(&tmp, hundred - pct);
320 for samp in samples {
323 } else if *samp < lo {
329 // Test vectors generated from R, using the script src/etc/stat-test-vectors.r.
336 use std::io::prelude::*;
339 macro_rules! assert_approx_eq {
340 ($a:expr, $b:expr) => ({
341 let (a, b) = (&$a, &$b);
342 assert!((*a - *b).abs() < 1.0e-6,
343 "{} is not approximately equal to {}", *a, *b);
347 fn check(samples: &[f64], summ: &Summary) {
349 let summ2 = Summary::new(samples);
351 let mut w = io::sink();
353 (write!(w, "\n")).unwrap();
355 assert_eq!(summ.sum, summ2.sum);
356 assert_eq!(summ.min, summ2.min);
357 assert_eq!(summ.max, summ2.max);
358 assert_eq!(summ.mean, summ2.mean);
359 assert_eq!(summ.median, summ2.median);
361 // We needed a few more digits to get exact equality on these
362 // but they're within float epsilon, which is 1.0e-6.
363 assert_approx_eq!(summ.var, summ2.var);
364 assert_approx_eq!(summ.std_dev, summ2.std_dev);
365 assert_approx_eq!(summ.std_dev_pct, summ2.std_dev_pct);
366 assert_approx_eq!(summ.median_abs_dev, summ2.median_abs_dev);
367 assert_approx_eq!(summ.median_abs_dev_pct, summ2.median_abs_dev_pct);
369 assert_eq!(summ.quartiles, summ2.quartiles);
370 assert_eq!(summ.iqr, summ2.iqr);
374 fn test_min_max_nan() {
375 let xs = &[1.0, 2.0, f64::NAN, 3.0, 4.0];
376 let summary = Summary::new(xs);
377 assert_eq!(summary.min, 1.0);
378 assert_eq!(summary.max, 4.0);
383 let val = &[958.0000000000, 924.0000000000];
384 let summ = &Summary {
385 sum: 1882.0000000000,
388 mean: 941.0000000000,
389 median: 941.0000000000,
391 std_dev: 24.0416305603,
392 std_dev_pct: 2.5549022912,
393 median_abs_dev: 25.2042000000,
394 median_abs_dev_pct: 2.6784484591,
395 quartiles: (932.5000000000, 941.0000000000, 949.5000000000),
401 fn test_norm10narrow() {
402 let val = &[966.0000000000,
412 let summ = &Summary {
413 sum: 9996.0000000000,
415 max: 1217.0000000000,
416 mean: 999.6000000000,
417 median: 970.5000000000,
418 var: 16050.7111111111,
419 std_dev: 126.6914010938,
420 std_dev_pct: 12.6742097933,
421 median_abs_dev: 102.2994000000,
422 median_abs_dev_pct: 10.5408964451,
423 quartiles: (956.7500000000, 970.5000000000, 1078.7500000000),
429 fn test_norm10medium() {
430 let val = &[954.0000000000,
440 let summ = &Summary {
441 sum: 8653.0000000000,
443 max: 1084.0000000000,
444 mean: 865.3000000000,
445 median: 911.5000000000,
446 var: 48628.4555555556,
447 std_dev: 220.5186059170,
448 std_dev_pct: 25.4846418487,
449 median_abs_dev: 195.7032000000,
450 median_abs_dev_pct: 21.4704552935,
451 quartiles: (771.0000000000, 911.5000000000, 1017.2500000000),
457 fn test_norm10wide() {
458 let val = &[505.0000000000,
468 let summ = &Summary {
469 sum: 9349.0000000000,
471 max: 1591.0000000000,
472 mean: 934.9000000000,
473 median: 913.5000000000,
474 var: 239208.9888888889,
475 std_dev: 489.0899599142,
476 std_dev_pct: 52.3146817750,
477 median_abs_dev: 611.5725000000,
478 median_abs_dev_pct: 66.9482758621,
479 quartiles: (567.2500000000, 913.5000000000, 1331.2500000000),
485 fn test_norm25verynarrow() {
486 let val = &[991.0000000000,
511 let summ = &Summary {
512 sum: 24926.0000000000,
514 max: 1040.0000000000,
515 mean: 997.0400000000,
516 median: 998.0000000000,
518 std_dev: 19.8294393937,
519 std_dev_pct: 1.9888308788,
520 median_abs_dev: 22.2390000000,
521 median_abs_dev_pct: 2.2283567134,
522 quartiles: (983.0000000000, 998.0000000000, 1013.0000000000),
529 let val = &[23.0000000000,
539 let summ = &Summary {
544 median: 11.5000000000,
546 std_dev: 16.9643416875,
547 std_dev_pct: 101.5828843560,
548 median_abs_dev: 13.3434000000,
549 median_abs_dev_pct: 116.0295652174,
550 quartiles: (4.2500000000, 11.5000000000, 22.5000000000),
557 let val = &[24.0000000000,
567 let summ = &Summary {
572 median: 24.5000000000,
574 std_dev: 19.5848580967,
575 std_dev_pct: 74.4671410520,
576 median_abs_dev: 22.9803000000,
577 median_abs_dev_pct: 93.7971428571,
578 quartiles: (9.5000000000, 24.5000000000, 36.5000000000),
585 let val = &[71.0000000000,
595 let summ = &Summary {
600 median: 22.0000000000,
602 std_dev: 21.4050876611,
603 std_dev_pct: 88.4507754589,
604 median_abs_dev: 21.4977000000,
605 median_abs_dev_pct: 97.7168181818,
606 quartiles: (7.7500000000, 22.0000000000, 35.0000000000),
613 let val = &[3.0000000000,
638 let summ = &Summary {
643 median: 19.0000000000,
645 std_dev: 24.5161851301,
646 std_dev_pct: 103.3565983562,
647 median_abs_dev: 19.2738000000,
648 median_abs_dev_pct: 101.4410526316,
649 quartiles: (6.0000000000, 19.0000000000, 31.0000000000),
656 let val = &[18.0000000000,
681 let summ = &Summary {
686 median: 20.0000000000,
688 std_dev: 4.5650848842,
689 std_dev_pct: 22.2037202539,
690 median_abs_dev: 5.9304000000,
691 median_abs_dev_pct: 29.6520000000,
692 quartiles: (17.0000000000, 20.0000000000, 24.0000000000),
698 fn test_pois25lambda30() {
699 let val = &[27.0000000000,
724 let summ = &Summary {
729 median: 32.0000000000,
731 std_dev: 5.1568724372,
732 std_dev_pct: 16.3814245145,
733 median_abs_dev: 5.9304000000,
734 median_abs_dev_pct: 18.5325000000,
735 quartiles: (28.0000000000, 32.0000000000, 34.0000000000),
741 fn test_pois25lambda40() {
742 let val = &[42.0000000000,
767 let summ = &Summary {
768 sum: 1019.0000000000,
772 median: 42.0000000000,
774 std_dev: 5.8685603004,
775 std_dev_pct: 14.3978417577,
776 median_abs_dev: 5.9304000000,
777 median_abs_dev_pct: 14.1200000000,
778 quartiles: (37.0000000000, 42.0000000000, 45.0000000000),
784 fn test_pois25lambda50() {
785 let val = &[45.0000000000,
810 let summ = &Summary {
811 sum: 1235.0000000000,
815 median: 50.0000000000,
817 std_dev: 5.6273143387,
818 std_dev_pct: 11.3913245723,
819 median_abs_dev: 4.4478000000,
820 median_abs_dev_pct: 8.8956000000,
821 quartiles: (44.0000000000, 50.0000000000, 52.0000000000),
828 let val = &[99.0000000000,
853 let summ = &Summary {
854 sum: 1242.0000000000,
858 median: 45.0000000000,
859 var: 1015.6433333333,
860 std_dev: 31.8691595957,
861 std_dev_pct: 64.1488719719,
862 median_abs_dev: 45.9606000000,
863 median_abs_dev_pct: 102.1346666667,
864 quartiles: (29.0000000000, 45.0000000000, 79.0000000000),
872 assert_eq!([0.5f64, 3.2321f64, 1.5678f64].sum(), 5.2999);
875 fn test_sum_f64_between_ints_that_sum_to_0() {
876 assert_eq!([1e30f64, 1.2f64, -1e30f64].sum(), 1.2);
886 pub fn sum_three_items(b: &mut Bencher) {
888 [1e20f64, 1.5f64, -1e20f64].sum();
892 pub fn sum_many_f64(b: &mut Bencher) {
893 let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60];
894 let v = (0..500).map(|i| nums[i % 5]).collect::<Vec<_>>();