rust/src/libtest/tests.rs
2019-12-22 17:42:47 -05:00

689 lines
21 KiB
Rust

use super::*;
use crate::{
bench::Bencher,
console::OutputLocation,
formatters::PrettyFormatter,
options::OutputFormat,
test::{
filter_tests,
parse_opts,
run_test,
DynTestFn,
DynTestName,
MetricMap,
RunIgnored,
RunStrategy,
ShouldPanic,
StaticTestName,
TestDesc,
TestDescAndFn,
TestOpts,
TrIgnored,
TrOk,
// FIXME (introduced by #65251)
// ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
// TestType, TrFailedMsg, TrIgnored, TrOk,
},
time::{TestTimeOptions, TimeThreshold},
};
use std::any::TypeId;
use std::sync::mpsc::channel;
use std::time::Duration;
impl TestOpts {
fn new() -> TestOpts {
TestOpts {
list: false,
filter: None,
filter_exact: false,
force_run_in_process: false,
exclude_should_panic: false,
run_ignored: RunIgnored::No,
run_tests: false,
bench_benchmarks: false,
logfile: None,
nocapture: false,
color: AutoColor,
format: OutputFormat::Pretty,
test_threads: None,
skip: vec![],
time_options: None,
options: Options::new(),
}
}
}
fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
vec![
TestDescAndFn {
desc: TestDesc {
name: StaticTestName("1"),
ignore: true,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(move || {})),
},
TestDescAndFn {
desc: TestDesc {
name: StaticTestName("2"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(move || {})),
},
]
}
#[test]
pub fn do_not_run_ignored_tests() {
fn f() {
panic!();
}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: true,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_ne!(result, TrOk);
}
#[test]
pub fn ignored_tests_result_in_ignored() {
fn f() {}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: true,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_eq!(result, TrIgnored);
}
// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic() {
fn f() {
panic!();
}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::Yes,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_eq!(result, TrOk);
}
// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_good_message() {
fn f() {
panic!("an error message");
}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::YesWithMessage("error message"),
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_eq!(result, TrOk);
}
// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_bad_message() {
use crate::tests::TrFailedMsg;
fn f() {
panic!("an error message");
}
let expected = "foobar";
let failed_msg = r#"panic did not contain expected string
panic message: `"an error message"`,
expected substring: `"foobar"`"#;
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::YesWithMessage(expected),
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
}
// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_non_string_message_type() {
use crate::tests::TrFailedMsg;
fn f() {
panic!(1i32);
}
let expected = "foobar";
let failed_msg = format!(
r#"expected panic with string value,
found non-string value: `{:?}`
expected substring: `"foobar"`"#,
TypeId::of::<i32>()
);
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::YesWithMessage(expected),
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_eq!(result, TrFailedMsg(failed_msg));
}
// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_but_succeeds() {
fn f() {}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::Yes,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let (tx, rx) = channel();
run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
assert_eq!(result, TrFailedMsg("test did not panic as expected".to_string()));
}
fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
fn f() {}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(f)),
};
let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
let test_opts = TestOpts { time_options, ..TestOpts::new() };
let (tx, rx) = channel();
run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let exec_time = rx.recv().unwrap().exec_time;
exec_time
}
#[test]
fn test_should_not_report_time() {
let exec_time = report_time_test_template(false);
assert!(exec_time.is_none());
}
#[test]
fn test_should_report_time() {
let exec_time = report_time_test_template(true);
assert!(exec_time.is_some());
}
fn time_test_failure_template(test_type: TestType) -> TestResult {
fn f() {}
let desc = TestDescAndFn {
desc: TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type,
},
testfn: DynTestFn(Box::new(f)),
};
// `Default` will initialize all the thresholds to 0 milliseconds.
let mut time_options = TestTimeOptions::default();
time_options.error_on_excess = true;
let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
let (tx, rx) = channel();
run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
let result = rx.recv().unwrap().result;
result
}
#[test]
fn test_error_on_exceed() {
let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
for test_type in types.iter() {
let result = time_test_failure_template(*test_type);
assert_eq!(result, TestResult::TrTimedFail);
}
// Check that for unknown tests thresholds aren't applied.
let result = time_test_failure_template(TestType::Unknown);
assert_eq!(result, TestResult::TrOk);
}
fn typed_test_desc(test_type: TestType) -> TestDesc {
TestDesc {
name: StaticTestName("whatever"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type,
}
}
fn test_exec_time(millis: u64) -> TestExecTime {
TestExecTime(Duration::from_millis(millis))
}
#[test]
fn test_time_options_threshold() {
let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
let options = TestTimeOptions {
error_on_excess: false,
colored: false,
unit_threshold: unit.clone(),
integration_threshold: integration.clone(),
doctest_threshold: doc.clone(),
};
let test_vector = [
(TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
(TestType::UnitTest, unit.warn.as_millis(), true, false),
(TestType::UnitTest, unit.critical.as_millis(), true, true),
(TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
(TestType::IntegrationTest, integration.warn.as_millis(), true, false),
(TestType::IntegrationTest, integration.critical.as_millis(), true, true),
(TestType::DocTest, doc.warn.as_millis() - 1, false, false),
(TestType::DocTest, doc.warn.as_millis(), true, false),
(TestType::DocTest, doc.critical.as_millis(), true, true),
];
for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
let test_desc = typed_test_desc(*test_type);
let exec_time = test_exec_time(*time as u64);
assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
}
}
#[test]
fn parse_ignored_flag() {
let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
let opts = parse_opts(&args).unwrap().unwrap();
assert_eq!(opts.run_ignored, RunIgnored::Only);
}
#[test]
fn parse_show_output_flag() {
let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
let opts = parse_opts(&args).unwrap().unwrap();
assert!(opts.options.display_output);
}
#[test]
fn parse_include_ignored_flag() {
let args = vec![
"progname".to_string(),
"filter".to_string(),
"-Zunstable-options".to_string(),
"--include-ignored".to_string(),
];
let opts = parse_opts(&args).unwrap().unwrap();
assert_eq!(opts.run_ignored, RunIgnored::Yes);
}
#[test]
pub fn filter_for_ignored_option() {
// When we run ignored tests the test filter should filter out all the
// unignored tests and flip the ignore flag on the rest to false
let mut opts = TestOpts::new();
opts.run_tests = true;
opts.run_ignored = RunIgnored::Only;
let tests = one_ignored_one_unignored_test();
let filtered = filter_tests(&opts, tests);
assert_eq!(filtered.len(), 1);
assert_eq!(filtered[0].desc.name.to_string(), "1");
assert!(!filtered[0].desc.ignore);
}
#[test]
pub fn run_include_ignored_option() {
// When we "--include-ignored" tests, the ignore flag should be set to false on
// all tests and no test filtered out
let mut opts = TestOpts::new();
opts.run_tests = true;
opts.run_ignored = RunIgnored::Yes;
let tests = one_ignored_one_unignored_test();
let filtered = filter_tests(&opts, tests);
assert_eq!(filtered.len(), 2);
assert!(!filtered[0].desc.ignore);
assert!(!filtered[1].desc.ignore);
}
#[test]
pub fn exclude_should_panic_option() {
let mut opts = TestOpts::new();
opts.run_tests = true;
opts.exclude_should_panic = true;
let mut tests = one_ignored_one_unignored_test();
tests.push(TestDescAndFn {
desc: TestDesc {
name: StaticTestName("3"),
ignore: false,
should_panic: ShouldPanic::Yes,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(move || {})),
});
let filtered = filter_tests(&opts, tests);
assert_eq!(filtered.len(), 2);
assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
}
#[test]
pub fn exact_filter_match() {
fn tests() -> Vec<TestDescAndFn> {
vec!["base", "base::test", "base::test1", "base::test2"]
.into_iter()
.map(|name| TestDescAndFn {
desc: TestDesc {
name: StaticTestName(name),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(move || {})),
})
.collect()
}
let substr =
filter_tests(&TestOpts { filter: Some("base".into()), ..TestOpts::new() }, tests());
assert_eq!(substr.len(), 4);
let substr = filter_tests(&TestOpts { filter: Some("bas".into()), ..TestOpts::new() }, tests());
assert_eq!(substr.len(), 4);
let substr =
filter_tests(&TestOpts { filter: Some("::test".into()), ..TestOpts::new() }, tests());
assert_eq!(substr.len(), 3);
let substr =
filter_tests(&TestOpts { filter: Some("base::test".into()), ..TestOpts::new() }, tests());
assert_eq!(substr.len(), 3);
let exact = filter_tests(
&TestOpts { filter: Some("base".into()), filter_exact: true, ..TestOpts::new() },
tests(),
);
assert_eq!(exact.len(), 1);
let exact = filter_tests(
&TestOpts { filter: Some("bas".into()), filter_exact: true, ..TestOpts::new() },
tests(),
);
assert_eq!(exact.len(), 0);
let exact = filter_tests(
&TestOpts { filter: Some("::test".into()), filter_exact: true, ..TestOpts::new() },
tests(),
);
assert_eq!(exact.len(), 0);
let exact = filter_tests(
&TestOpts { filter: Some("base::test".into()), filter_exact: true, ..TestOpts::new() },
tests(),
);
assert_eq!(exact.len(), 1);
}
#[test]
pub fn sort_tests() {
let mut opts = TestOpts::new();
opts.run_tests = true;
let names = vec![
"sha1::test".to_string(),
"isize::test_to_str".to_string(),
"isize::test_pow".to_string(),
"test::do_not_run_ignored_tests".to_string(),
"test::ignored_tests_result_in_ignored".to_string(),
"test::first_free_arg_should_be_a_filter".to_string(),
"test::parse_ignored_flag".to_string(),
"test::parse_include_ignored_flag".to_string(),
"test::filter_for_ignored_option".to_string(),
"test::run_include_ignored_option".to_string(),
"test::sort_tests".to_string(),
];
let tests = {
fn testfn() {}
let mut tests = Vec::new();
for name in &names {
let test = TestDescAndFn {
desc: TestDesc {
name: DynTestName((*name).clone()),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
},
testfn: DynTestFn(Box::new(testfn)),
};
tests.push(test);
}
tests
};
let filtered = filter_tests(&opts, tests);
let expected = vec![
"isize::test_pow".to_string(),
"isize::test_to_str".to_string(),
"sha1::test".to_string(),
"test::do_not_run_ignored_tests".to_string(),
"test::filter_for_ignored_option".to_string(),
"test::first_free_arg_should_be_a_filter".to_string(),
"test::ignored_tests_result_in_ignored".to_string(),
"test::parse_ignored_flag".to_string(),
"test::parse_include_ignored_flag".to_string(),
"test::run_include_ignored_option".to_string(),
"test::sort_tests".to_string(),
];
for (a, b) in expected.iter().zip(filtered) {
assert_eq!(*a, b.desc.name.to_string());
}
}
#[test]
pub fn test_metricmap_compare() {
let mut m1 = MetricMap::new();
let mut m2 = MetricMap::new();
m1.insert_metric("in-both-noise", 1000.0, 200.0);
m2.insert_metric("in-both-noise", 1100.0, 200.0);
m1.insert_metric("in-first-noise", 1000.0, 2.0);
m2.insert_metric("in-second-noise", 1000.0, 2.0);
m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
}
#[test]
pub fn test_bench_once_no_iter() {
fn f(_: &mut Bencher) {}
bench::run_once(f);
}
#[test]
pub fn test_bench_once_iter() {
fn f(b: &mut Bencher) {
b.iter(|| {})
}
bench::run_once(f);
}
#[test]
pub fn test_bench_no_iter() {
fn f(_: &mut Bencher) {}
let (tx, rx) = channel();
let desc = TestDesc {
name: StaticTestName("f"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
};
crate::bench::benchmark(desc, tx, true, f);
rx.recv().unwrap();
}
#[test]
pub fn test_bench_iter() {
fn f(b: &mut Bencher) {
b.iter(|| {})
}
let (tx, rx) = channel();
let desc = TestDesc {
name: StaticTestName("f"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
};
crate::bench::benchmark(desc, tx, true, f);
rx.recv().unwrap();
}
#[test]
fn should_sort_failures_before_printing_them() {
let test_a = TestDesc {
name: StaticTestName("a"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
};
let test_b = TestDesc {
name: StaticTestName("b"),
ignore: false,
should_panic: ShouldPanic::No,
allow_fail: false,
test_type: TestType::Unknown,
};
let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
let st = console::ConsoleTestState {
log_out: None,
total: 0,
passed: 0,
failed: 0,
ignored: 0,
allowed_fail: 0,
filtered_out: 0,
measured: 0,
metrics: MetricMap::new(),
failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
options: Options::new(),
not_failures: Vec::new(),
time_failures: Vec::new(),
};
out.write_failures(&st).unwrap();
let s = match out.output_location() {
&OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
&OutputLocation::Pretty(_) => unreachable!(),
};
let apos = s.find("a").unwrap();
let bpos = s.find("b").unwrap();
assert!(apos < bpos);
}