1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 #![allow(missing_docs)]
12 #![allow(deprecated)] // Float
14 use std::cmp::Ordering::{self, Less, Greater, Equal};
16 use std::num::{Float, FromPrimitive};
18 fn local_cmp<T:Float>(x: T, y: T) -> Ordering {
19 // arbitrarily decide that NaNs are larger than everything.
22 } else if x.is_nan() {
33 fn local_sort<T: Float>(v: &mut [T]) {
34 v.sort_by(|x: &T, y: &T| local_cmp(*x, *y));
37 /// Trait that provides simple descriptive statistics on a univariate set of numeric samples.
38 pub trait Stats <T: Float + FromPrimitive> {
40 /// Sum of the samples.
42 /// Note: this method sacrifices performance at the altar of accuracy
43 /// Depends on IEEE-754 arithmetic guarantees. See proof of correctness at:
44 /// ["Adaptive Precision Floating-Point Arithmetic and Fast Robust Geometric Predicates"]
45 /// (http://www.cs.cmu.edu/~quake-papers/robust-arithmetic.ps)
46 /// *Discrete & Computational Geometry 18*, 3 (Oct 1997), 305-363, Shewchuk J.R.
49 /// Minimum value of the samples.
52 /// Maximum value of the samples.
55 /// Arithmetic mean (average) of the samples: sum divided by sample-count.
57 /// See: https://en.wikipedia.org/wiki/Arithmetic_mean
60 /// Median of the samples: value separating the lower half of the samples from the higher half.
61 /// Equal to `self.percentile(50.0)`.
63 /// See: https://en.wikipedia.org/wiki/Median
64 fn median(&self) -> T;
66 /// Variance of the samples: bias-corrected mean of the squares of the differences of each
67 /// sample from the sample mean. Note that this calculates the _sample variance_ rather than the
68 /// population variance, which is assumed to be unknown. It therefore corrects the `(n-1)/n`
69 /// bias that would appear if we calculated a population variance, by dividing by `(n-1)` rather
72 /// See: https://en.wikipedia.org/wiki/Variance
75 /// Standard deviation: the square root of the sample variance.
77 /// Note: this is not a robust statistic for non-normal distributions. Prefer the
78 /// `median_abs_dev` for unknown distributions.
80 /// See: https://en.wikipedia.org/wiki/Standard_deviation
81 fn std_dev(&self) -> T;
83 /// Standard deviation as a percent of the mean value. See `std_dev` and `mean`.
85 /// Note: this is not a robust statistic for non-normal distributions. Prefer the
86 /// `median_abs_dev_pct` for unknown distributions.
87 fn std_dev_pct(&self) -> T;
89 /// Scaled median of the absolute deviations of each sample from the sample median. This is a
90 /// robust (distribution-agnostic) estimator of sample variability. Use this in preference to
91 /// `std_dev` if you cannot assume your sample is normally distributed. Note that this is scaled
92 /// by the constant `1.4826` to allow its use as a consistent estimator for the standard
95 /// See: http://en.wikipedia.org/wiki/Median_absolute_deviation
96 fn median_abs_dev(&self) -> T;
98 /// Median absolute deviation as a percent of the median. See `median_abs_dev` and `median`.
99 fn median_abs_dev_pct(&self) -> T;
101 /// Percentile: the value below which `pct` percent of the values in `self` fall. For example,
102 /// percentile(95.0) will return the value `v` such that 95% of the samples `s` in `self`
103 /// satisfy `s <= v`.
105 /// Calculated by linear interpolation between closest ranks.
107 /// See: http://en.wikipedia.org/wiki/Percentile
108 fn percentile(&self, pct: T) -> T;
110 /// Quartiles of the sample: three values that divide the sample into four equal groups, each
111 /// with 1/4 of the data. The middle value is the median. See `median` and `percentile`. This
112 /// function may calculate the 3 quartiles more efficiently than 3 calls to `percentile`, but
113 /// is otherwise equivalent.
115 /// See also: https://en.wikipedia.org/wiki/Quartile
116 fn quartiles(&self) -> (T,T,T);
118 /// Inter-quartile range: the difference between the 25th percentile (1st quartile) and the 75th
119 /// percentile (3rd quartile). See `quartiles`.
121 /// See also: https://en.wikipedia.org/wiki/Interquartile_range
125 /// Extracted collection of all the summary statistics of a sample set.
126 #[derive(Clone, PartialEq)]
127 #[allow(missing_docs)]
128 pub struct Summary<T> {
137 pub median_abs_dev: T,
138 pub median_abs_dev_pct: T,
139 pub quartiles: (T,T,T),
143 impl<T: Float + FromPrimitive> Summary<T> {
144 /// Construct a new summary of a sample set.
145 pub fn new(samples: &[T]) -> Summary<T> {
150 mean: samples.mean(),
151 median: samples.median(),
153 std_dev: samples.std_dev(),
154 std_dev_pct: samples.std_dev_pct(),
155 median_abs_dev: samples.median_abs_dev(),
156 median_abs_dev_pct: samples.median_abs_dev_pct(),
157 quartiles: samples.quartiles(),
163 impl<T: Float + FromPrimitive> Stats<T> for [T] {
164 // FIXME #11059 handle NaN, inf and overflow
166 let mut partials = vec![];
171 // This inner loop applies `hi`/`lo` summation to each
172 // partial so that the list of partial sums remains exact.
173 for i in 0..partials.len() {
174 let mut y: T = partials[i];
175 if x.abs() < y.abs() {
176 mem::swap(&mut x, &mut y);
178 // Rounded `x+y` is stored in `hi` with round-off stored in
179 // `lo`. Together `hi+lo` are exactly equal to `x+y`.
181 let lo = y - (hi - x);
182 if lo != Float::zero() {
188 if j >= partials.len() {
192 partials.truncate(j+1);
195 let zero: T = Float::zero();
196 partials.iter().fold(zero, |p, q| p + *q)
200 assert!(self.len() != 0);
201 self.iter().fold(self[0], |p, q| p.min(*q))
205 assert!(self.len() != 0);
206 self.iter().fold(self[0], |p, q| p.max(*q))
209 fn mean(&self) -> T {
210 assert!(self.len() != 0);
211 self.sum() / FromPrimitive::from_usize(self.len()).unwrap()
214 fn median(&self) -> T {
215 self.percentile(FromPrimitive::from_usize(50).unwrap())
222 let mean = self.mean();
223 let mut v: T = Float::zero();
228 // NB: this is _supposed to be_ len-1, not len. If you
229 // change it back to len, you will be calculating a
230 // population variance, not a sample variance.
231 let denom = FromPrimitive::from_usize(self.len()-1).unwrap();
236 fn std_dev(&self) -> T {
240 fn std_dev_pct(&self) -> T {
241 let hundred = FromPrimitive::from_usize(100).unwrap();
242 (self.std_dev() / self.mean()) * hundred
245 fn median_abs_dev(&self) -> T {
246 let med = self.median();
247 let abs_devs: Vec<T> = self.iter().map(|&v| (med - v).abs()).collect();
248 // This constant is derived by smarter statistics brains than me, but it is
249 // consistent with how R and other packages treat the MAD.
250 let number = FromPrimitive::from_f64(1.4826).unwrap();
251 abs_devs.median() * number
254 fn median_abs_dev_pct(&self) -> T {
255 let hundred = FromPrimitive::from_usize(100).unwrap();
256 (self.median_abs_dev() / self.median()) * hundred
259 fn percentile(&self, pct: T) -> T {
260 let mut tmp = self.to_vec();
261 local_sort(&mut tmp);
262 percentile_of_sorted(&tmp, pct)
265 fn quartiles(&self) -> (T,T,T) {
266 let mut tmp = self.to_vec();
267 local_sort(&mut tmp);
268 let first = FromPrimitive::from_usize(25).unwrap();
269 let a = percentile_of_sorted(&tmp, first);
270 let secound = FromPrimitive::from_usize(50).unwrap();
271 let b = percentile_of_sorted(&tmp, secound);
272 let third = FromPrimitive::from_usize(75).unwrap();
273 let c = percentile_of_sorted(&tmp, third);
278 let (a,_,c) = self.quartiles();
284 // Helper function: extract a value representing the `pct` percentile of a sorted sample-set, using
285 // linear interpolation. If samples are not sorted, return nonsensical value.
286 fn percentile_of_sorted<T: Float + FromPrimitive>(sorted_samples: &[T],
288 assert!(sorted_samples.len() != 0);
289 if sorted_samples.len() == 1 {
290 return sorted_samples[0];
292 let zero: T = Float::zero();
293 assert!(zero <= pct);
294 let hundred = FromPrimitive::from_usize(100).unwrap();
295 assert!(pct <= hundred);
297 return sorted_samples[sorted_samples.len() - 1];
299 let length = FromPrimitive::from_usize(sorted_samples.len() - 1).unwrap();
300 let rank = (pct / hundred) * length;
301 let lrank = rank.floor();
302 let d = rank - lrank;
303 let n = lrank.to_usize().unwrap();
304 let lo = sorted_samples[n];
305 let hi = sorted_samples[n+1];
310 /// Winsorize a set of samples, replacing values above the `100-pct` percentile and below the `pct`
311 /// percentile with those percentiles themselves. This is a way of minimizing the effect of
312 /// outliers, at the cost of biasing the sample. It differs from trimming in that it does not
313 /// change the number of samples, just changes the values of those that are outliers.
315 /// See: http://en.wikipedia.org/wiki/Winsorising
316 pub fn winsorize<T: Float + FromPrimitive>(samples: &mut [T], pct: T) {
317 let mut tmp = samples.to_vec();
318 local_sort(&mut tmp);
319 let lo = percentile_of_sorted(&tmp, pct);
320 let hundred: T = FromPrimitive::from_usize(100).unwrap();
321 let hi = percentile_of_sorted(&tmp, hundred-pct);
322 for samp in samples {
325 } else if *samp < lo {
331 // Test vectors generated from R, using the script src/etc/stat-test-vectors.r.
337 use std::old_io::{self, Writer};
340 macro_rules! assert_approx_eq {
341 ($a:expr, $b:expr) => ({
343 let (a, b) = (&$a, &$b);
344 assert!((*a - *b).abs() < 1.0e-6,
345 "{} is not approximately equal to {}", *a, *b);
349 fn check(samples: &[f64], summ: &Summary<f64>) {
351 let summ2 = Summary::new(samples);
353 let mut w = old_io::stdout();
355 (write!(w, "\n")).unwrap();
357 assert_eq!(summ.sum, summ2.sum);
358 assert_eq!(summ.min, summ2.min);
359 assert_eq!(summ.max, summ2.max);
360 assert_eq!(summ.mean, summ2.mean);
361 assert_eq!(summ.median, summ2.median);
363 // We needed a few more digits to get exact equality on these
364 // but they're within float epsilon, which is 1.0e-6.
365 assert_approx_eq!(summ.var, summ2.var);
366 assert_approx_eq!(summ.std_dev, summ2.std_dev);
367 assert_approx_eq!(summ.std_dev_pct, summ2.std_dev_pct);
368 assert_approx_eq!(summ.median_abs_dev, summ2.median_abs_dev);
369 assert_approx_eq!(summ.median_abs_dev_pct, summ2.median_abs_dev_pct);
371 assert_eq!(summ.quartiles, summ2.quartiles);
372 assert_eq!(summ.iqr, summ2.iqr);
376 fn test_min_max_nan() {
377 let xs = &[1.0, 2.0, f64::NAN, 3.0, 4.0];
378 let summary = Summary::new(xs);
379 assert_eq!(summary.min, 1.0);
380 assert_eq!(summary.max, 4.0);
389 let summ = &Summary {
390 sum: 1882.0000000000,
393 mean: 941.0000000000,
394 median: 941.0000000000,
396 std_dev: 24.0416305603,
397 std_dev_pct: 2.5549022912,
398 median_abs_dev: 25.2042000000,
399 median_abs_dev_pct: 2.6784484591,
400 quartiles: (932.5000000000,941.0000000000,949.5000000000),
406 fn test_norm10narrow() {
419 let summ = &Summary {
420 sum: 9996.0000000000,
422 max: 1217.0000000000,
423 mean: 999.6000000000,
424 median: 970.5000000000,
425 var: 16050.7111111111,
426 std_dev: 126.6914010938,
427 std_dev_pct: 12.6742097933,
428 median_abs_dev: 102.2994000000,
429 median_abs_dev_pct: 10.5408964451,
430 quartiles: (956.7500000000,970.5000000000,1078.7500000000),
436 fn test_norm10medium() {
449 let summ = &Summary {
450 sum: 8653.0000000000,
452 max: 1084.0000000000,
453 mean: 865.3000000000,
454 median: 911.5000000000,
455 var: 48628.4555555556,
456 std_dev: 220.5186059170,
457 std_dev_pct: 25.4846418487,
458 median_abs_dev: 195.7032000000,
459 median_abs_dev_pct: 21.4704552935,
460 quartiles: (771.0000000000,911.5000000000,1017.2500000000),
466 fn test_norm10wide() {
479 let summ = &Summary {
480 sum: 9349.0000000000,
482 max: 1591.0000000000,
483 mean: 934.9000000000,
484 median: 913.5000000000,
485 var: 239208.9888888889,
486 std_dev: 489.0899599142,
487 std_dev_pct: 52.3146817750,
488 median_abs_dev: 611.5725000000,
489 median_abs_dev_pct: 66.9482758621,
490 quartiles: (567.2500000000,913.5000000000,1331.2500000000),
496 fn test_norm25verynarrow() {
524 let summ = &Summary {
525 sum: 24926.0000000000,
527 max: 1040.0000000000,
528 mean: 997.0400000000,
529 median: 998.0000000000,
531 std_dev: 19.8294393937,
532 std_dev_pct: 1.9888308788,
533 median_abs_dev: 22.2390000000,
534 median_abs_dev_pct: 2.2283567134,
535 quartiles: (983.0000000000,998.0000000000,1013.0000000000),
554 let summ = &Summary {
559 median: 11.5000000000,
561 std_dev: 16.9643416875,
562 std_dev_pct: 101.5828843560,
563 median_abs_dev: 13.3434000000,
564 median_abs_dev_pct: 116.0295652174,
565 quartiles: (4.2500000000,11.5000000000,22.5000000000),
584 let summ = &Summary {
589 median: 24.5000000000,
591 std_dev: 19.5848580967,
592 std_dev_pct: 74.4671410520,
593 median_abs_dev: 22.9803000000,
594 median_abs_dev_pct: 93.7971428571,
595 quartiles: (9.5000000000,24.5000000000,36.5000000000),
614 let summ = &Summary {
619 median: 22.0000000000,
621 std_dev: 21.4050876611,
622 std_dev_pct: 88.4507754589,
623 median_abs_dev: 21.4977000000,
624 median_abs_dev_pct: 97.7168181818,
625 quartiles: (7.7500000000,22.0000000000,35.0000000000),
659 let summ = &Summary {
664 median: 19.0000000000,
666 std_dev: 24.5161851301,
667 std_dev_pct: 103.3565983562,
668 median_abs_dev: 19.2738000000,
669 median_abs_dev_pct: 101.4410526316,
670 quartiles: (6.0000000000,19.0000000000,31.0000000000),
704 let summ = &Summary {
709 median: 20.0000000000,
711 std_dev: 4.5650848842,
712 std_dev_pct: 22.2037202539,
713 median_abs_dev: 5.9304000000,
714 median_abs_dev_pct: 29.6520000000,
715 quartiles: (17.0000000000,20.0000000000,24.0000000000),
721 fn test_pois25lambda30() {
749 let summ = &Summary {
754 median: 32.0000000000,
756 std_dev: 5.1568724372,
757 std_dev_pct: 16.3814245145,
758 median_abs_dev: 5.9304000000,
759 median_abs_dev_pct: 18.5325000000,
760 quartiles: (28.0000000000,32.0000000000,34.0000000000),
766 fn test_pois25lambda40() {
794 let summ = &Summary {
795 sum: 1019.0000000000,
799 median: 42.0000000000,
801 std_dev: 5.8685603004,
802 std_dev_pct: 14.3978417577,
803 median_abs_dev: 5.9304000000,
804 median_abs_dev_pct: 14.1200000000,
805 quartiles: (37.0000000000,42.0000000000,45.0000000000),
811 fn test_pois25lambda50() {
839 let summ = &Summary {
840 sum: 1235.0000000000,
844 median: 50.0000000000,
846 std_dev: 5.6273143387,
847 std_dev_pct: 11.3913245723,
848 median_abs_dev: 4.4478000000,
849 median_abs_dev_pct: 8.8956000000,
850 quartiles: (44.0000000000,50.0000000000,52.0000000000),
884 let summ = &Summary {
885 sum: 1242.0000000000,
889 median: 45.0000000000,
890 var: 1015.6433333333,
891 std_dev: 31.8691595957,
892 std_dev_pct: 64.1488719719,
893 median_abs_dev: 45.9606000000,
894 median_abs_dev_pct: 102.1346666667,
895 quartiles: (29.0000000000,45.0000000000,79.0000000000),
903 assert_eq!([0.5f64, 3.2321f64, 1.5678f64].sum(), 5.2999);
906 fn test_sum_f64_between_ints_that_sum_to_0() {
907 assert_eq!([1e30f64, 1.2f64, -1e30f64].sum(), 1.2);
917 pub fn sum_three_items(b: &mut Bencher) {
919 [1e20f64, 1.5f64, -1e20f64].sum();
923 pub fn sum_many_f64(b: &mut Bencher) {
924 let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60];
925 let v = (0..500).map(|i| nums[i%5]).collect::<Vec<_>>();