1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
17 // NB: this can probably be rewritten in terms of num::Num
18 // to be less f64-specific.
20 /// Trait that provides simple descriptive statistics on a univariate set of numeric samples.
23 /// Sum of the samples.
26 /// Minimum value of the samples.
29 /// Maximum value of the samples.
32 /// Arithmetic mean (average) of the samples: sum divided by sample-count.
34 /// See: https://en.wikipedia.org/wiki/Arithmetic_mean
37 /// Median of the samples: value separating the lower half of the samples from the higher half.
38 /// Equal to `self.percentile(50.0)`.
40 /// See: https://en.wikipedia.org/wiki/Median
41 fn median(self) -> f64;
43 /// Variance of the samples: bias-corrected mean of the squares of the differences of each
44 /// sample from the sample mean. Note that this calculates the _sample variance_ rather than the
45 /// population variance, which is assumed to be unknown. It therefore corrects the `(n-1)/n`
46 /// bias that would appear if we calculated a population variance, by dividing by `(n-1)` rather
49 /// See: https://en.wikipedia.org/wiki/Variance
52 /// Standard deviation: the square root of the sample variance.
54 /// Note: this is not a robust statistic for non-normal distributions. Prefer the
55 /// `median_abs_dev` for unknown distributions.
57 /// See: https://en.wikipedia.org/wiki/Standard_deviation
58 fn std_dev(self) -> f64;
60 /// Standard deviation as a percent of the mean value. See `std_dev` and `mean`.
62 /// Note: this is not a robust statistic for non-normal distributions. Prefer the
63 /// `median_abs_dev_pct` for unknown distributions.
64 fn std_dev_pct(self) -> f64;
66 /// Scaled median of the absolute deviations of each sample from the sample median. This is a
67 /// robust (distribution-agnostic) estimator of sample variability. Use this in preference to
68 /// `std_dev` if you cannot assume your sample is normally distributed. Note that this is scaled
69 /// by the constant `1.4826` to allow its use as a consistent estimator for the standard
72 /// See: http://en.wikipedia.org/wiki/Median_absolute_deviation
73 fn median_abs_dev(self) -> f64;
75 /// Median absolute deviation as a percent of the median. See `median_abs_dev` and `median`.
76 fn median_abs_dev_pct(self) -> f64;
78 /// Percentile: the value below which `pct` percent of the values in `self` fall. For example,
79 /// percentile(95.0) will return the value `v` such that that 95% of the samples `s` in `self`
82 /// Calculated by linear interpolation between closest ranks.
84 /// See: http://en.wikipedia.org/wiki/Percentile
85 fn percentile(self, pct: f64) -> f64;
87 /// Quartiles of the sample: three values that divide the sample into four equal groups, each
88 /// with 1/4 of the data. The middle value is the median. See `median` and `percentile`. This
89 /// function may calculate the 3 quartiles more efficiently than 3 calls to `percentile`, but
90 /// is otherwise equivalent.
92 /// See also: https://en.wikipedia.org/wiki/Quartile
93 fn quartiles(self) -> (f64,f64,f64);
95 /// Inter-quartile range: the difference between the 25th percentile (1st quartile) and the 75th
96 /// percentile (3rd quartile). See `quartiles`.
98 /// See also: https://en.wikipedia.org/wiki/Interquartile_range
102 /// Extracted collection of all the summary statistics of a sample set.
103 #[deriving(Clone, Eq)]
114 median_abs_dev_pct: f64,
115 quartiles: (f64,f64,f64),
121 /// Construct a new summary of a sample set.
122 pub fn new(samples: &[f64]) -> Summary {
127 mean: samples.mean(),
128 median: samples.median(),
130 std_dev: samples.std_dev(),
131 std_dev_pct: samples.std_dev_pct(),
132 median_abs_dev: samples.median_abs_dev(),
133 median_abs_dev_pct: samples.median_abs_dev_pct(),
134 quartiles: samples.quartiles(),
140 impl<'self> Stats for &'self [f64] {
142 fn sum(self) -> f64 {
143 self.iter().fold(0.0, |p,q| p + *q)
146 fn min(self) -> f64 {
147 assert!(self.len() != 0);
148 self.iter().fold(self[0], |p,q| cmp::min(p, *q))
151 fn max(self) -> f64 {
152 assert!(self.len() != 0);
153 self.iter().fold(self[0], |p,q| cmp::max(p, *q))
156 fn mean(self) -> f64 {
157 assert!(self.len() != 0);
158 self.sum() / (self.len() as f64)
161 fn median(self) -> f64 {
162 self.percentile(50.0)
165 fn var(self) -> f64 {
169 let mean = self.mean();
171 for s in self.iter() {
175 // NB: this is _supposed to be_ len-1, not len. If you
176 // change it back to len, you will be calculating a
177 // population variance, not a sample variance.
178 v/((self.len()-1) as f64)
182 fn std_dev(self) -> f64 {
186 fn std_dev_pct(self) -> f64 {
187 (self.std_dev() / self.mean()) * 100.0
190 fn median_abs_dev(self) -> f64 {
191 let med = self.median();
192 let abs_devs = self.map(|&v| num::abs(med - v));
193 // This constant is derived by smarter statistics brains than me, but it is
194 // consistent with how R and other packages treat the MAD.
195 abs_devs.median() * 1.4826
198 fn median_abs_dev_pct(self) -> f64 {
199 (self.median_abs_dev() / self.median()) * 100.0
202 fn percentile(self, pct: f64) -> f64 {
203 let mut tmp = self.to_owned();
205 percentile_of_sorted(tmp, pct)
208 fn quartiles(self) -> (f64,f64,f64) {
209 let mut tmp = self.to_owned();
211 let a = percentile_of_sorted(tmp, 25.0);
212 let b = percentile_of_sorted(tmp, 50.0);
213 let c = percentile_of_sorted(tmp, 75.0);
217 fn iqr(self) -> f64 {
218 let (a,_,c) = self.quartiles();
224 // Helper function: extract a value representing the `pct` percentile of a sorted sample-set, using
225 // linear interpolation. If samples are not sorted, return nonsensical value.
226 priv fn percentile_of_sorted(sorted_samples: &[f64],
228 assert!(sorted_samples.len() != 0);
229 if sorted_samples.len() == 1 {
230 return sorted_samples[0];
233 assert!(pct <= 100.0);
235 return sorted_samples[sorted_samples.len() - 1];
237 let rank = (pct / 100.0) * ((sorted_samples.len() - 1) as f64);
238 let lrank = rank.floor();
239 let d = rank - lrank;
240 let n = lrank as uint;
241 let lo = sorted_samples[n];
242 let hi = sorted_samples[n+1];
247 /// Winsorize a set of samples, replacing values above the `100-pct` percentile and below the `pct`
248 /// percentile with those percentiles themselves. This is a way of minimizing the effect of
249 /// outliers, at the cost of biasing the sample. It differs from trimming in that it does not
250 /// change the number of samples, just changes the values of those that are outliers.
252 /// See: http://en.wikipedia.org/wiki/Winsorising
253 pub fn winsorize(samples: &mut [f64], pct: f64) {
254 let mut tmp = samples.to_owned();
256 let lo = percentile_of_sorted(tmp, pct);
257 let hi = percentile_of_sorted(tmp, 100.0-pct);
258 for samp in samples.mut_iter() {
261 } else if *samp < lo {
267 /// Render writes the min, max and quartiles of the provided `Summary` to the provided `Writer`.
268 pub fn write_5_number_summary(w: @io::Writer, s: &Summary) {
269 let (q1,q2,q3) = s.quartiles;
270 w.write_str(fmt!("(min=%f, q1=%f, med=%f, q3=%f, max=%f)",
278 /// Render a boxplot to the provided writer. The boxplot shows the min, max and quartiles of the
279 /// provided `Summary` (thus includes the mean) and is scaled to display within the range of the
280 /// nearest multiple-of-a-power-of-ten above and below the min and max of possible values, and
281 /// target `width_hint` characters of display (though it will be wider if necessary).
283 /// As an example, the summary with 5-number-summary `(min=15, q1=17, med=20, q3=24, max=31)` might
287 /// 10 | [--****#******----------] | 40
290 pub fn write_boxplot(w: @io::Writer, s: &Summary, width_hint: uint) {
292 let (q1,q2,q3) = s.quartiles;
294 let lomag = (10.0_f64).pow(&s.min.log10().floor());
295 let himag = (10.0_f64).pow(&(s.max.log10().floor()));
296 let lo = (s.min / lomag).floor() * lomag;
297 let hi = (s.max / himag).ceil() * himag;
301 let lostr = lo.to_str();
302 let histr = hi.to_str();
304 let overhead_width = lostr.len() + histr.len() + 4;
305 let range_width = width_hint - overhead_width;;
306 let char_step = range / (range_width as f64);
315 while c < range_width && v < s.min {
322 while c < range_width && v < q1 {
327 while c < range_width && v < q2 {
334 while c < range_width && v < q3 {
339 while c < range_width && v < s.max {
345 while c < range_width {
356 /// Returns a HashMap with the number of occurences of every element in the
357 /// sequence that the iterator exposes.
358 pub fn freq_count<T: Iterator<U>, U: Eq+Hash>(mut iter: T) -> hashmap::HashMap<U, uint> {
359 let mut map = hashmap::HashMap::new::<U, uint>();
361 map.insert_or_update_with(elem, 1, |_, count| *count += 1);
366 // Test vectors generated from R, using the script src/etc/stat-test-vectors.r.
373 use stats::write_5_number_summary;
374 use stats::write_boxplot;
377 fn check(samples: &[f64], summ: &Summary) {
379 let summ2 = Summary::new(samples);
381 let w = io::stdout();
383 write_5_number_summary(w, &summ2);
385 write_boxplot(w, &summ2, 50);
388 assert_eq!(summ.sum, summ2.sum);
389 assert_eq!(summ.min, summ2.min);
390 assert_eq!(summ.max, summ2.max);
391 assert_eq!(summ.mean, summ2.mean);
392 assert_eq!(summ.median, summ2.median);
394 // We needed a few more digits to get exact equality on these
395 // but they're within float epsilon, which is 1.0e-6.
396 assert_approx_eq!(summ.var, summ2.var);
397 assert_approx_eq!(summ.std_dev, summ2.std_dev);
398 assert_approx_eq!(summ.std_dev_pct, summ2.std_dev_pct);
399 assert_approx_eq!(summ.median_abs_dev, summ2.median_abs_dev);
400 assert_approx_eq!(summ.median_abs_dev_pct, summ2.median_abs_dev_pct);
402 assert_eq!(summ.quartiles, summ2.quartiles);
403 assert_eq!(summ.iqr, summ2.iqr);
412 let summ = &Summary {
413 sum: 1882.0000000000,
416 mean: 941.0000000000,
417 median: 941.0000000000,
419 std_dev: 24.0416305603,
420 std_dev_pct: 2.5549022912,
421 median_abs_dev: 25.2042000000,
422 median_abs_dev_pct: 2.6784484591,
423 quartiles: (932.5000000000,941.0000000000,949.5000000000),
429 fn test_norm10narrow() {
442 let summ = &Summary {
443 sum: 9996.0000000000,
445 max: 1217.0000000000,
446 mean: 999.6000000000,
447 median: 970.5000000000,
448 var: 16050.7111111111,
449 std_dev: 126.6914010938,
450 std_dev_pct: 12.6742097933,
451 median_abs_dev: 102.2994000000,
452 median_abs_dev_pct: 10.5408964451,
453 quartiles: (956.7500000000,970.5000000000,1078.7500000000),
459 fn test_norm10medium() {
472 let summ = &Summary {
473 sum: 8653.0000000000,
475 max: 1084.0000000000,
476 mean: 865.3000000000,
477 median: 911.5000000000,
478 var: 48628.4555555556,
479 std_dev: 220.5186059170,
480 std_dev_pct: 25.4846418487,
481 median_abs_dev: 195.7032000000,
482 median_abs_dev_pct: 21.4704552935,
483 quartiles: (771.0000000000,911.5000000000,1017.2500000000),
489 fn test_norm10wide() {
502 let summ = &Summary {
503 sum: 9349.0000000000,
505 max: 1591.0000000000,
506 mean: 934.9000000000,
507 median: 913.5000000000,
508 var: 239208.9888888889,
509 std_dev: 489.0899599142,
510 std_dev_pct: 52.3146817750,
511 median_abs_dev: 611.5725000000,
512 median_abs_dev_pct: 66.9482758621,
513 quartiles: (567.2500000000,913.5000000000,1331.2500000000),
519 fn test_norm25verynarrow() {
547 let summ = &Summary {
548 sum: 24926.0000000000,
550 max: 1040.0000000000,
551 mean: 997.0400000000,
552 median: 998.0000000000,
554 std_dev: 19.8294393937,
555 std_dev_pct: 1.9888308788,
556 median_abs_dev: 22.2390000000,
557 median_abs_dev_pct: 2.2283567134,
558 quartiles: (983.0000000000,998.0000000000,1013.0000000000),
577 let summ = &Summary {
582 median: 11.5000000000,
584 std_dev: 16.9643416875,
585 std_dev_pct: 101.5828843560,
586 median_abs_dev: 13.3434000000,
587 median_abs_dev_pct: 116.0295652174,
588 quartiles: (4.2500000000,11.5000000000,22.5000000000),
607 let summ = &Summary {
612 median: 24.5000000000,
614 std_dev: 19.5848580967,
615 std_dev_pct: 74.4671410520,
616 median_abs_dev: 22.9803000000,
617 median_abs_dev_pct: 93.7971428571,
618 quartiles: (9.5000000000,24.5000000000,36.5000000000),
637 let summ = &Summary {
642 median: 22.0000000000,
644 std_dev: 21.4050876611,
645 std_dev_pct: 88.4507754589,
646 median_abs_dev: 21.4977000000,
647 median_abs_dev_pct: 97.7168181818,
648 quartiles: (7.7500000000,22.0000000000,35.0000000000),
682 let summ = &Summary {
687 median: 19.0000000000,
689 std_dev: 24.5161851301,
690 std_dev_pct: 103.3565983562,
691 median_abs_dev: 19.2738000000,
692 median_abs_dev_pct: 101.4410526316,
693 quartiles: (6.0000000000,19.0000000000,31.0000000000),
727 let summ = &Summary {
732 median: 20.0000000000,
734 std_dev: 4.5650848842,
735 std_dev_pct: 22.2037202539,
736 median_abs_dev: 5.9304000000,
737 median_abs_dev_pct: 29.6520000000,
738 quartiles: (17.0000000000,20.0000000000,24.0000000000),
744 fn test_pois25lambda30() {
772 let summ = &Summary {
777 median: 32.0000000000,
779 std_dev: 5.1568724372,
780 std_dev_pct: 16.3814245145,
781 median_abs_dev: 5.9304000000,
782 median_abs_dev_pct: 18.5325000000,
783 quartiles: (28.0000000000,32.0000000000,34.0000000000),
789 fn test_pois25lambda40() {
817 let summ = &Summary {
818 sum: 1019.0000000000,
822 median: 42.0000000000,
824 std_dev: 5.8685603004,
825 std_dev_pct: 14.3978417577,
826 median_abs_dev: 5.9304000000,
827 median_abs_dev_pct: 14.1200000000,
828 quartiles: (37.0000000000,42.0000000000,45.0000000000),
834 fn test_pois25lambda50() {
862 let summ = &Summary {
863 sum: 1235.0000000000,
867 median: 50.0000000000,
869 std_dev: 5.6273143387,
870 std_dev_pct: 11.3913245723,
871 median_abs_dev: 4.4478000000,
872 median_abs_dev_pct: 8.8956000000,
873 quartiles: (44.0000000000,50.0000000000,52.0000000000),
907 let summ = &Summary {
908 sum: 1242.0000000000,
912 median: 45.0000000000,
913 var: 1015.6433333333,
914 std_dev: 31.8691595957,
915 std_dev_pct: 64.1488719719,
916 median_abs_dev: 45.9606000000,
917 median_abs_dev_pct: 102.1346666667,
918 quartiles: (29.0000000000,45.0000000000,79.0000000000),