Add a custom run-coverage mode to compiletest
This commit is contained in:
Zalathar
parent
a42bbd0edd
commit
22e119bbac
@ -66,6 +66,7 @@ string_enum! {
|
||||
JsDocTest => "js-doc-test",
|
||||
MirOpt => "mir-opt",
|
||||
Assembly => "assembly",
|
||||
RunCoverage => "run-coverage",
|
||||
}
|
||||
}
|
||||
|
||||
@ -626,6 +627,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
|
||||
UI_STDERR_64,
|
||||
UI_STDERR_32,
|
||||
UI_STDERR_16,
|
||||
UI_COVERAGE,
|
||||
];
|
||||
pub const UI_STDERR: &str = "stderr";
|
||||
pub const UI_STDOUT: &str = "stdout";
|
||||
@ -635,6 +637,7 @@ pub const UI_RUN_STDOUT: &str = "run.stdout";
|
||||
pub const UI_STDERR_64: &str = "64bit.stderr";
|
||||
pub const UI_STDERR_32: &str = "32bit.stderr";
|
||||
pub const UI_STDERR_16: &str = "16bit.stderr";
|
||||
pub const UI_COVERAGE: &str = "coverage";
|
||||
|
||||
/// Absolute path to the directory where all output for all tests in the given
|
||||
/// `relative_dir` group should reside. Example:
|
||||
|
||||
@ -612,10 +612,25 @@ pub fn line_directive<'line>(
|
||||
}
|
||||
|
||||
fn iter_header<R: Read>(testfile: &Path, rdr: R, it: &mut dyn FnMut(Option<&str>, &str, usize)) {
|
||||
iter_header_extra(testfile, rdr, &[], it)
|
||||
}
|
||||
|
||||
fn iter_header_extra(
|
||||
testfile: &Path,
|
||||
rdr: impl Read,
|
||||
extra_directives: &[&str],
|
||||
it: &mut dyn FnMut(Option<&str>, &str, usize),
|
||||
) {
|
||||
if testfile.is_dir() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Process any extra directives supplied by the caller (e.g. because they
|
||||
// are implied by the test mode), with a dummy line number of 0.
|
||||
for directive in extra_directives {
|
||||
it(None, directive, 0);
|
||||
}
|
||||
|
||||
let comment = if testfile.extension().map(|e| e == "rs") == Some(true) { "//" } else { "#" };
|
||||
|
||||
let mut rdr = BufReader::new(rdr);
|
||||
@ -894,7 +909,27 @@ pub fn make_test_description<R: Read>(
|
||||
let mut ignore_message = None;
|
||||
let mut should_fail = false;
|
||||
|
||||
iter_header(path, src, &mut |revision, ln, line_number| {
|
||||
let extra_directives: &[&str] = match config.mode {
|
||||
// The run-coverage tests are treated as having these extra directives,
|
||||
// without needing to specify them manually in every test file.
|
||||
// (Some of the comments below have been copied over from
|
||||
// `tests/run-make/coverage-reports/Makefile`.)
|
||||
Mode::RunCoverage => {
|
||||
&[
|
||||
"needs-profiler-support",
|
||||
// FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
|
||||
// properly. Since we only have GCC on the CI ignore the test for now.
|
||||
"ignore-windows-gnu",
|
||||
// FIXME(pietroalbini): this test currently does not work on cross-compiled
|
||||
// targets because remote-test is not capable of sending back the *.profraw
|
||||
// files generated by the LLVM instrumentation.
|
||||
"ignore-cross-compile",
|
||||
]
|
||||
}
|
||||
_ => &[],
|
||||
};
|
||||
|
||||
iter_header_extra(path, src, extra_directives, &mut |revision, ln, line_number| {
|
||||
if revision.is_some() && revision != cfg {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs
|
||||
use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
|
||||
use crate::common::{CompareMode, FailMode, PassMode};
|
||||
use crate::common::{Config, TestPaths};
|
||||
use crate::common::{Pretty, RunPassValgrind};
|
||||
use crate::common::{UI_RUN_STDERR, UI_RUN_STDOUT};
|
||||
use crate::common::{Pretty, RunCoverage, RunPassValgrind};
|
||||
use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT};
|
||||
use crate::compute_diff::{write_diff, write_filtered_diff};
|
||||
use crate::errors::{self, Error, ErrorKind};
|
||||
use crate::header::TestProps;
|
||||
@ -253,6 +253,7 @@ impl<'test> TestCx<'test> {
|
||||
MirOpt => self.run_mir_opt_test(),
|
||||
Assembly => self.run_assembly_test(),
|
||||
JsDocTest => self.run_js_doc_test(),
|
||||
RunCoverage => self.run_coverage_test(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -465,6 +466,184 @@ impl<'test> TestCx<'test> {
|
||||
}
|
||||
}
|
||||
|
||||
fn run_coverage_test(&self) {
|
||||
let should_run = self.run_if_enabled();
|
||||
let proc_res = self.compile_test(should_run, Emit::None);
|
||||
|
||||
if !proc_res.status.success() {
|
||||
self.fatal_proc_rec("compilation failed!", &proc_res);
|
||||
}
|
||||
drop(proc_res);
|
||||
|
||||
if let WillExecute::Disabled = should_run {
|
||||
return;
|
||||
}
|
||||
|
||||
let profraw_path = self.output_base_dir().join("default.profraw");
|
||||
let profdata_path = self.output_base_dir().join("default.profdata");
|
||||
|
||||
// Delete any existing profraw/profdata files to rule out unintended
|
||||
// interference between repeated test runs.
|
||||
if profraw_path.exists() {
|
||||
std::fs::remove_file(&profraw_path).unwrap();
|
||||
}
|
||||
if profdata_path.exists() {
|
||||
std::fs::remove_file(&profdata_path).unwrap();
|
||||
}
|
||||
|
||||
let proc_res = self.exec_compiled_test_general(
|
||||
&[("LLVM_PROFILE_FILE", &profraw_path.to_str().unwrap())],
|
||||
false,
|
||||
);
|
||||
if self.props.failure_status.is_some() {
|
||||
self.check_correct_failure_status(&proc_res);
|
||||
} else if !proc_res.status.success() {
|
||||
self.fatal_proc_rec("test run failed!", &proc_res);
|
||||
}
|
||||
drop(proc_res);
|
||||
|
||||
// Run `llvm-profdata merge` to index the raw coverage output.
|
||||
let proc_res = self.run_llvm_tool("llvm-profdata", |cmd| {
|
||||
cmd.args(["merge", "--sparse", "--output"]);
|
||||
cmd.arg(&profdata_path);
|
||||
cmd.arg(&profraw_path);
|
||||
});
|
||||
if !proc_res.status.success() {
|
||||
self.fatal_proc_rec("llvm-profdata merge failed!", &proc_res);
|
||||
}
|
||||
drop(proc_res);
|
||||
|
||||
// Run `llvm-cov show` to produce a coverage report in text format.
|
||||
let proc_res = self.run_llvm_tool("llvm-cov", |cmd| {
|
||||
cmd.args(["show", "--format=text", "--show-line-counts-or-regions"]);
|
||||
|
||||
cmd.arg("--Xdemangler");
|
||||
cmd.arg(self.config.rust_demangler_path.as_ref().unwrap());
|
||||
|
||||
cmd.arg("--instr-profile");
|
||||
cmd.arg(&profdata_path);
|
||||
|
||||
cmd.arg("--object");
|
||||
cmd.arg(&self.make_exe_name());
|
||||
});
|
||||
if !proc_res.status.success() {
|
||||
self.fatal_proc_rec("llvm-cov show failed!", &proc_res);
|
||||
}
|
||||
|
||||
let kind = UI_COVERAGE;
|
||||
|
||||
let expected_coverage = self.load_expected_output(kind);
|
||||
let normalized_actual_coverage =
|
||||
self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| {
|
||||
self.fatal_proc_rec(&err, &proc_res);
|
||||
});
|
||||
|
||||
let coverage_errors = self.compare_output(
|
||||
kind,
|
||||
&normalized_actual_coverage,
|
||||
&expected_coverage,
|
||||
self.props.compare_output_lines_by_subset,
|
||||
);
|
||||
|
||||
if coverage_errors > 0 {
|
||||
self.fatal_proc_rec(
|
||||
&format!("{} errors occurred comparing coverage output.", coverage_errors),
|
||||
&proc_res,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn run_llvm_tool(&self, name: &str, configure_cmd_fn: impl FnOnce(&mut Command)) -> ProcRes {
|
||||
let tool_path = self
|
||||
.config
|
||||
.llvm_bin_dir
|
||||
.as_ref()
|
||||
.expect("this test expects the LLVM bin dir to be available")
|
||||
.join(name);
|
||||
|
||||
let mut cmd = Command::new(tool_path);
|
||||
configure_cmd_fn(&mut cmd);
|
||||
|
||||
let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`"));
|
||||
|
||||
let proc_res = ProcRes {
|
||||
status: output.status,
|
||||
stdout: String::from_utf8(output.stdout).unwrap(),
|
||||
stderr: String::from_utf8(output.stderr).unwrap(),
|
||||
cmdline: format!("{cmd:?}"),
|
||||
};
|
||||
self.dump_output(&proc_res.stdout, &proc_res.stderr);
|
||||
|
||||
proc_res
|
||||
}
|
||||
|
||||
fn normalize_coverage_output(&self, coverage: &str) -> Result<String, String> {
|
||||
let normalized = self.normalize_output(coverage, &[]);
|
||||
|
||||
let mut lines = normalized.lines().collect::<Vec<_>>();
|
||||
|
||||
Self::sort_coverage_subviews(&mut lines)?;
|
||||
|
||||
let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
|
||||
Ok(joined_lines)
|
||||
}
|
||||
|
||||
fn sort_coverage_subviews(coverage_lines: &mut Vec<&str>) -> Result<(), String> {
|
||||
let mut output_lines = Vec::new();
|
||||
|
||||
// We accumulate a list of zero or more "subviews", where each
|
||||
// subview is a list of one or more lines.
|
||||
let mut subviews: Vec<Vec<&str>> = Vec::new();
|
||||
|
||||
fn flush<'a>(subviews: &mut Vec<Vec<&'a str>>, output_lines: &mut Vec<&'a str>) {
|
||||
if subviews.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Take and clear the list of accumulated subviews.
|
||||
let mut subviews = std::mem::take(subviews);
|
||||
|
||||
// The last "subview" should be just a boundary line on its own,
|
||||
// so exclude it when sorting the other subviews.
|
||||
let except_last = subviews.len() - 1;
|
||||
(&mut subviews[..except_last]).sort();
|
||||
|
||||
for view in subviews {
|
||||
for line in view {
|
||||
output_lines.push(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (line, line_num) in coverage_lines.iter().zip(1..) {
|
||||
if line.starts_with(" ------------------") {
|
||||
// This is a subview boundary line, so start a new subview.
|
||||
subviews.push(vec![line]);
|
||||
} else if line.starts_with(" |") {
|
||||
// Add this line to the current subview.
|
||||
subviews
|
||||
.last_mut()
|
||||
.ok_or(format!(
|
||||
"unexpected subview line outside of a subview on line {line_num}"
|
||||
))?
|
||||
.push(line);
|
||||
} else {
|
||||
// This line is not part of a subview, so sort and print any
|
||||
// accumulated subviews, and then print the line as-is.
|
||||
flush(&mut subviews, &mut output_lines);
|
||||
output_lines.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
flush(&mut subviews, &mut output_lines);
|
||||
assert!(subviews.is_empty());
|
||||
|
||||
assert_eq!(output_lines.len(), coverage_lines.len());
|
||||
*coverage_lines = output_lines;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_pretty_test(&self) {
|
||||
if self.props.pp_exact.is_some() {
|
||||
logv(self.config, "testing for exact pretty-printing".to_owned());
|
||||
@ -1822,6 +2001,7 @@ impl<'test> TestCx<'test> {
|
||||
|| self.is_vxworks_pure_static()
|
||||
|| self.config.target.contains("bpf")
|
||||
|| !self.config.target_cfg().dynamic_linking
|
||||
|| self.config.mode == RunCoverage
|
||||
{
|
||||
// We primarily compile all auxiliary libraries as dynamic libraries
|
||||
// to avoid code size bloat and large binaries as much as possible
|
||||
@ -1832,6 +2012,10 @@ impl<'test> TestCx<'test> {
|
||||
// dynamic libraries so we just go back to building a normal library. Note,
|
||||
// however, that for MUSL if the library is built with `force_host` then
|
||||
// it's ok to be a dylib as the host should always support dylibs.
|
||||
//
|
||||
// Coverage tests want static linking by default so that coverage
|
||||
// mappings in auxiliary libraries can be merged into the final
|
||||
// executable.
|
||||
(false, Some("lib"))
|
||||
} else {
|
||||
(true, Some("dylib"))
|
||||
@ -2009,6 +2193,10 @@ impl<'test> TestCx<'test> {
|
||||
}
|
||||
}
|
||||
DebugInfo => { /* debuginfo tests must be unoptimized */ }
|
||||
RunCoverage => {
|
||||
// Coverage reports are affected by optimization level, and
|
||||
// the current snapshots assume no optimization by default.
|
||||
}
|
||||
_ => {
|
||||
rustc.arg("-O");
|
||||
}
|
||||
@ -2075,6 +2263,9 @@ impl<'test> TestCx<'test> {
|
||||
|
||||
rustc.arg(dir_opt);
|
||||
}
|
||||
RunCoverage => {
|
||||
rustc.arg("-Cinstrument-coverage");
|
||||
}
|
||||
RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake
|
||||
| CodegenUnits | JsDocTest | Assembly => {
|
||||
// do not use JSON output
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user