mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add allow_fail test attribute

Browse Source 
This change allows the user to add an `#[allow_fail]` attribute to
tests that will cause the test to compile & run, but if the test fails
it will not cause the entire test run to fail. The test output will
show the failure, but in yellow instead of red, and also indicate that
it was an allowed failure.
This commit is contained in:
Paul Woolcock 2017-05-24 13:58:37 -04:00
parent 229d0d3266
commit 60dd83ea85
7 changed files with 81 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/librustdoc/html/markdown.rs
View File

@ -769,7 +769,7 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position
block_info.should_panic, block_info.no_run,
block_info.ignore, block_info.test_harness,
block_info.compile_fail, block_info.error_codes,
line, filename);
line, filename, block_info.allow_fail);
} else {
tests.add_old_test(text, filename);
}
@ -859,7 +859,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Sp
block_info.should_panic, block_info.no_run,
block_info.ignore, block_info.test_harness,
block_info.compile_fail, block_info.error_codes,
line, filename);
line, filename, block_info.allow_fail);
prev_offset = offset;
}
Event::Start(Tag::Header(level)) => {
@ -889,6 +889,7 @@ struct LangString {
test_harness: bool,
compile_fail: bool,
error_codes: Vec<String>,
allow_fail: bool,
}
impl LangString {
@ -902,6 +903,7 @@ impl LangString {
test_harness: false,
compile_fail: false,
error_codes: Vec::new(),
allow_fail: false,
}
}
@ -930,6 +932,7 @@ impl LangString {
}
"no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; }
"ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; }
"allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; }
"rust" => { data.rust = true; seen_rust_tags = true; }
"test_harness" => {
data.test_harness = true;
@ -1118,7 +1121,7 @@ mod tests {
fn test_lang_string_parse() {
fn t(s: &str,
should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
compile_fail: bool, error_codes: Vec<String>) {
compile_fail: bool, allow_fail: bool, error_codes: Vec<String>) {
assert_eq!(LangString::parse(s), LangString {
should_panic: should_panic,
no_run: no_run,
@ -1128,25 +1131,27 @@ mod tests {
compile_fail: compile_fail,
error_codes: error_codes,
original: s.to_owned(),
allow_fail: allow_fail,
})
}
// marker | should_panic| no_run| ignore| rust | test_harness| compile_fail
// | error_codes
t("", false, false, false, true, false, false, Vec::new());
t("rust", false, false, false, true, false, false, Vec::new());
t("sh", false, false, false, false, false, false, Vec::new());
t("ignore", false, false, true, true, false, false, Vec::new());
t("should_panic", true, false, false, true, false, false, Vec::new());
t("no_run", false, true, false, true, false, false, Vec::new());
t("test_harness", false, false, false, true, true, false, Vec::new());
t("compile_fail", false, true, false, true, false, true, Vec::new());
t("{.no_run .example}", false, true, false, true, false, false, Vec::new());
t("{.sh .should_panic}", true, false, false, false, false, false, Vec::new());
t("{.example .rust}", false, false, false, true, false, false, Vec::new());
t("{.test_harness .rust}", false, false, false, true, true, false, Vec::new());
t("text, no_run", false, true, false, false, false, false, Vec::new());
t("text,no_run", false, true, false, false, false, false, Vec::new());
// | allow_fail | error_codes
t("", false, false, false, true, false, false, false, Vec::new());
t("rust", false, false, false, true, false, false, false, Vec::new());
t("sh", false, false, false, false, false, false, false, Vec::new());
t("ignore", false, false, true, true, false, false, false, Vec::new());
t("should_panic", true, false, false, true, false, false, false, Vec::new());
t("no_run", false, true, false, true, false, false, false, Vec::new());
t("test_harness", false, false, false, true, true, false, false, Vec::new());
t("compile_fail", false, true, false, true, false, true, false, Vec::new());
t("allow_fail", false, false, false, true, false, false, true, Vec::new());
t("{.no_run .example}", false, true, false, true, false, false, false, Vec::new());
t("{.sh .should_panic}", true, false, false, false, false, false, false, Vec::new());
t("{.example .rust}", false, false, false, true, false, false, false, Vec::new());
t("{.test_harness .rust}", false, false, false, true, true, false, false, Vec::new());
t("text, no_run", false, true, false, false, false, false, false, Vec::new());
t("text,no_run", false, true, false, false, false, false, false, Vec::new());
}
#[test]

3
src/librustdoc/test.rs
View File

@ -467,7 +467,7 @@ impl Collector {
pub fn add_test(&mut self, test: String,
should_panic: bool, no_run: bool, should_ignore: bool,
as_test_harness: bool, compile_fail: bool, error_codes: Vec<String>,
line: usize, filename: String) {
line: usize, filename: String, allow_fail: bool) {
let name = self.generate_name(line, &filename);
// to be removed when hoedown is removed
if self.render_type == RenderType::Pulldown {
@ -499,6 +499,7 @@ impl Collector {
ignore: should_ignore,
// compiler failures are test failures
should_panic: testing::ShouldPanic::No,
allow_fail: allow_fail,
},
testfn: testing::DynTestFn(box move |()| {
let panic = io::set_panic(None);

1
src/libsyntax/feature_gate.rs
View File

@ -534,6 +534,7 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
("derive", Normal, Ungated),
("should_panic", Normal, Ungated),
("ignore", Normal, Ungated),
("allow_fail", Normal, Ungated),
("no_implicit_prelude", Normal, Ungated),
("reexport_test_harness_main", Normal, Ungated),
("link_args", Normal, Ungated),

14
src/libsyntax/test.rs
View File

@ -52,7 +52,8 @@ struct Test {
path: Vec<Ident> ,
bench: bool,
ignore: bool,
should_panic: ShouldPanic
should_panic: ShouldPanic,
allow_fail: bool,
}
struct TestCtxt<'a> {
@ -133,7 +134,8 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> {
path: self.cx.path.clone(),
bench: is_bench_fn(&self.cx, &i),
ignore: is_ignored(&i),
should_panic: should_panic(&i, &self.cx)
should_panic: should_panic(&i, &self.cx),
allow_fail: is_allowed_fail(&i),
};
self.cx.testfns.push(test);
self.tests.push(i.ident);
@ -383,6 +385,10 @@ fn is_ignored(i: &ast::Item) -> bool {
i.attrs.iter().any(|attr| attr.check_name("ignore"))
}
fn is_allowed_fail(i: &ast::Item) -> bool {
i.attrs.iter().any(|attr| attr.check_name("allow_fail"))
}
fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic {
match i.attrs.iter().find(|attr| attr.check_name("should_panic")) {
Some(attr) => {
@ -668,6 +674,7 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
}
}
};
let allow_fail_expr = ecx.expr_bool(span, test.allow_fail);
// self::test::TestDesc { ... }
let desc_expr = ecx.expr_struct(
@ -675,7 +682,8 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
test_path("TestDesc"),
vec![field("name", name_expr),
field("ignore", ignore_expr),
field("should_panic", fail_expr)]);
field("should_panic", fail_expr),
field("allow_fail", allow_fail_expr)]);
let mut visible_path = match cx.toplevel_reexport {

23
src/libtest/lib.rs
View File

@ -212,6 +212,7 @@ pub struct TestDesc {
pub name: TestName,
pub ignore: bool,
pub should_panic: ShouldPanic,
pub allow_fail: bool,
}
#[derive(Clone)]
@ -523,6 +524,7 @@ pub enum TestResult {
TrFailed,
TrFailedMsg(String),
TrIgnored,
TrAllowedFail,
TrMetrics(MetricMap),
TrBench(BenchSamples),
}
@ -543,6 +545,7 @@ struct ConsoleTestState<T> {
passed: usize,
failed: usize,
ignored: usize,
allowed_fail: usize,
filtered_out: usize,
measured: usize,
metrics: MetricMap,
@ -572,6 +575,7 @@ impl<T: Write> ConsoleTestState<T> {
passed: 0,
failed: 0,
ignored: 0,
allowed_fail: 0,
filtered_out: 0,
measured: 0,
metrics: MetricMap::new(),
@ -594,6 +598,10 @@ impl<T: Write> ConsoleTestState<T> {
self.write_short_result("ignored", "i", term::color::YELLOW)
}
pub fn write_allowed_fail(&mut self) -> io::Result<()> {
self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW)
}
pub fn write_metric(&mut self) -> io::Result<()> {
self.write_pretty("metric", term::color::CYAN)
}
@ -669,6 +677,7 @@ impl<T: Write> ConsoleTestState<T> {
TrOk => self.write_ok(),
TrFailed | TrFailedMsg(_) => self.write_failed(),
TrIgnored => self.write_ignored(),
TrAllowedFail => self.write_allowed_fail(),
TrMetrics(ref mm) => {
self.write_metric()?;
self.write_plain(&format!(": {}\n", mm.fmt_metrics()))
@ -702,6 +711,7 @@ impl<T: Write> ConsoleTestState<T> {
TrFailed => "failed".to_owned(),
TrFailedMsg(ref msg) => format!("failed: {}", msg),
TrIgnored => "ignored".to_owned(),
TrAllowedFail => "failed (allowed)".to_owned(),
TrMetrics(ref mm) => mm.fmt_metrics(),
TrBench(ref bs) => fmt_bench_samples(bs),
},
@ -761,7 +771,7 @@ impl<T: Write> ConsoleTestState<T> {
}
pub fn write_run_finish(&mut self) -> io::Result<bool> {
assert!(self.passed + self.failed + self.ignored + self.measured == self.total);
assert!(self.passed + self.failed + self.ignored + self.measured + self.allowed_fail == self.total);
if self.options.display_output {
self.write_outputs()?;
@ -778,9 +788,10 @@ impl<T: Write> ConsoleTestState<T> {
} else {
self.write_pretty("FAILED", term::color::RED)?;
}
let s = format!(". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
let s = format!(". {} passed; {} failed; {} allowed to fail; {} ignored; {} measured; {} filtered out\n\n",
self.passed,
self.failed,
self.allowed_fail,
self.ignored,
self.measured,
self.filtered_out);
@ -891,6 +902,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Resu
st.not_failures.push((test, stdout));
}
TrIgnored => st.ignored += 1,
TrAllowedFail => st.allowed_fail += 1,
TrMetrics(mm) => {
let tname = test.name;
let MetricMap(mm) = mm;
@ -1471,8 +1483,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> Tes
.unwrap_or(false) {
TrOk
} else {
TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
if desc.allow_fail {
TrAllowedFail
} else {
TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
}
},
_ if desc.allow_fail => TrAllowedFail,
_ => TrFailed,
}
}

23
src/test/run-pass/test-allow-fail-attr.rs Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// compile-flags: --test
#[test]
#[allow_fail]
fn test1() {
panic!();
}
#[test]
#[allow_fail]
fn test2() {
assert!(true);
}

1
src/tools/compiletest/src/main.rs
View File

@ -476,6 +476,7 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn
name: make_test_name(config, testpaths),
ignore: ignore,
should_panic: should_panic,
allow_fail: false,
},
testfn: make_test_closure(config, testpaths),
}