From 77823c907b272836108175dee546fb846b925465 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Thu, 11 Jul 2013 17:05:23 -0700 Subject: [PATCH] extra: add tests for test::MetricMap, MetricDiff, ratchet. --- src/libextra/test.rs | 110 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 5 deletions(-) diff --git a/src/libextra/test.rs b/src/libextra/test.rs index 96ca4296768..98338b4eaed 100644 --- a/src/libextra/test.rs +++ b/src/libextra/test.rs @@ -98,6 +98,7 @@ pub struct Metric { pub struct MetricMap(TreeMap<~str,Metric>); /// Analysis of a single change in metric +#[deriving(Eq)] pub enum MetricChange { LikelyNoise, MetricAdded, @@ -774,8 +775,13 @@ pub fn save(&self, p: &Path) { json::to_pretty_writer(f, &self.to_json()); } - /// Compare against another MetricMap - pub fn compare_to_old(&self, old: MetricMap, + /// Compare against another MetricMap. Optionally compare all + /// measurements in the maps using the provided `noise_pct` as a + /// percentage of each value to consider noise. If `None`, each + /// measurement's noise threshold is independently chosen as the + /// maximum of that measurement's recorded noise quantity in either + /// map. + pub fn compare_to_old(&self, old: &MetricMap, noise_pct: Option) -> MetricDiff { let mut diff : MetricDiff = TreeMap::new(); for old.iter().advance |(k, vold)| { @@ -790,7 +796,7 @@ pub fn compare_to_old(&self, old: MetricMap, if delta.abs() < noise { LikelyNoise } else { - let pct = delta.abs() / v.value * 100.0; + let pct = delta.abs() / vold.value * 100.0; if vold.noise < 0.0 { // When 'noise' is negative, it means we want // to see deltas that go up over time, and can @@ -857,7 +863,7 @@ pub fn ratchet(&self, p: &Path, pct: Option) -> (MetricDiff, bool) { MetricMap::new() }; - let diff : MetricDiff = self.compare_to_old(old, pct); + let diff : MetricDiff = self.compare_to_old(&old, pct); let ok = do diff.iter().all() |(_, v)| { match *v { Regression(_) => false, @@ -1006,13 +1012,16 @@ pub fn benchmark(f: &fn(&mut BenchHarness)) -> BenchSamples { mod tests { use test::{TrFailed, TrIgnored, TrOk, filter_tests, parse_opts, TestDesc, TestDescAndFn, + Metric, MetricMap, MetricAdded, MetricRemoved, + Improvement, Regression, LikelyNoise, StaticTestName, DynTestName, DynTestFn}; use test::{TestOpts, run_test}; use std::either; use std::comm::{stream, SharedChan}; - use std::option; use std::vec; + use tempfile; + use std::os; #[test] pub fn do_not_run_ignored_tests() { @@ -1208,4 +1217,95 @@ fn testfn() { } } } } + + #[test] + pub fn test_metricmap_compare() { + let mut m1 = MetricMap::new(); + let mut m2 = MetricMap::new(); + m1.insert_metric("in-both-noise", 1000.0, 200.0); + m2.insert_metric("in-both-noise", 1100.0, 200.0); + + m1.insert_metric("in-first-noise", 1000.0, 2.0); + m2.insert_metric("in-second-noise", 1000.0, 2.0); + + m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0); + m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0); + + m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0); + m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0); + + m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0); + m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0); + + m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0); + m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0); + + let diff1 = m2.compare_to_old(&m1, None); + + assert_eq!(*(diff1.find(&~"in-both-noise").get()), LikelyNoise); + assert_eq!(*(diff1.find(&~"in-first-noise").get()), MetricRemoved); + assert_eq!(*(diff1.find(&~"in-second-noise").get()), MetricAdded); + assert_eq!(*(diff1.find(&~"in-both-want-downwards-but-regressed").get()), Regression(100.0)); + assert_eq!(*(diff1.find(&~"in-both-want-downwards-and-improved").get()), Improvement(50.0)); + assert_eq!(*(diff1.find(&~"in-both-want-upwards-but-regressed").get()), Regression(50.0)); + assert_eq!(*(diff1.find(&~"in-both-want-upwards-and-improved").get()), Improvement(100.0)); + assert_eq!(diff1.len(), 7); + + let diff2 = m2.compare_to_old(&m1, Some(200.0)); + + assert_eq!(*(diff2.find(&~"in-both-noise").get()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-first-noise").get()), MetricRemoved); + assert_eq!(*(diff2.find(&~"in-second-noise").get()), MetricAdded); + assert_eq!(*(diff2.find(&~"in-both-want-downwards-but-regressed").get()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-both-want-downwards-and-improved").get()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-both-want-upwards-but-regressed").get()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-both-want-upwards-and-improved").get()), LikelyNoise); + assert_eq!(diff2.len(), 7); + } + + pub fn ratchet_test() { + + let dpth = tempfile::mkdtemp(&os::tmpdir(), + "test-ratchet").expect("missing test for ratchet"); + let pth = dpth.push("ratchet.json"); + + let mut m1 = MetricMap::new(); + m1.insert_metric("runtime", 1000.0, 2.0); + m1.insert_metric("throughput", 50.0, 2.0); + + let mut m2 = MetricMap::new(); + m2.insert_metric("runtime", 1100.0, 2.0); + m2.insert_metric("throughput", 50.0, 2.0); + + m1.save(&pth); + + // Ask for a ratchet that should fail to advance. + let (diff1, ok1) = m2.ratchet(&pth, None); + assert_eq!(ok1, false); + assert_eq!(diff1.len(), 2); + assert_eq!(*(diff1.find(&~"runtime").get()), Regression(10.0)); + assert_eq!(*(diff1.find(&~"throughput").get()), LikelyNoise); + + // Check that it was not rewritten. + let m3 = MetricMap::load(&pth); + assert_eq!(m3.len(), 2); + assert_eq!(*(m3.find(&~"runtime").get()), Metric { value: 1000.0, noise: 2.0 }); + assert_eq!(*(m3.find(&~"throughput").get()), Metric { value: 50.0, noise: 2.0 }); + + // Ask for a ratchet with an explicit noise-percentage override, + // that should advance. + let (diff2, ok2) = m2.ratchet(&pth, Some(10.0)); + assert_eq!(ok2, true); + assert_eq!(diff2.len(), 2); + assert_eq!(*(diff2.find(&~"runtime").get()), LikelyNoise); + assert_eq!(*(diff2.find(&~"throughput").get()), LikelyNoise); + + // Check that it was rewritten. + let m4 = MetricMap::load(&pth); + assert_eq!(m4.len(), 2); + assert_eq!(*(m4.find(&~"runtime").get()), Metric { value: 1100.0, noise: 2.0 }); + assert_eq!(*(m4.find(&~"throughput").get()), Metric { value: 50.0, noise: 2.0 }); + + os::remove_dir_recursive(&dpth); + } }