Rollup merge of #112300 - Zalathar:run-coverage, r=wesleywiser

Convert `run-make/coverage-reports` tests to use a custom compiletest mode

I was frustrated by the fact that most of the coverage tests are glued together with makefiles and shell scripts, so I tried my hand at converting most of them over to a newly-implemented `run-coverage` mode/suite in compiletest.

This ~~*mostly*~~ resolves #85009, ~~though I've left a small number of the existing tests as-is because they would require more work to fix/support~~.

---

I had time to go back and add support for the more troublesome tests that I had initially skipped over, so this PR now manages to completely get rid of `run-make/coverage-reports`.

---

The patches are arranged as follows:

- Declare the new mode/suite in bootstrap
- Small changes to compiletest that will be used by the new mode
- Implement the new mode in compiletest
- Migrate most of the tests over
- Add more code to bootstrap and compiletest to support the remaining tests
- Migrate the remaining tests (with some temporary hacks to avoid re-blessing them)
- Remove the temporary hacks and re-bless the migrated tests
- Remove the unused remnants of `run-make/coverage-reports`

src/bootstrap/builder.rs

@@ -686,6 +686,7 @@ macro_rules! describe {
                 test::Tidy,
                 test::Ui,
                 test::RunPassValgrind,
+                test::RunCoverage,
                 test::MirOpt,
                 test::Codegen,
                 test::CodegenUnits,
@@ -694,6 +695,7 @@ macro_rules! describe {
                 test::Debuginfo,
                 test::UiFullDeps,
                 test::Rustdoc,
+                test::RunCoverageRustdoc,
                 test::Pretty,
                 test::Crate,
                 test::CrateLibrustc,

src/bootstrap/test.rs

@@ -1319,6 +1319,13 @@ fn run(self, builder: &Builder<'_>) {
 
 default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" });
 
+host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" });
+host_test!(RunCoverageRustdoc {
+    path: "tests/run-coverage-rustdoc",
+    mode: "run-coverage",
+    suite: "run-coverage-rustdoc"
+});
+
 // For the mir-opt suite we do not use macros, as we need custom behavior when blessing.
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct MirOpt {
@@ -1503,6 +1510,7 @@ fn run(self, builder: &Builder<'_>) {
             || (mode == "ui" && is_rustdoc)
             || mode == "js-doc-test"
             || mode == "rustdoc-json"
+            || suite == "run-coverage-rustdoc"
         {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
         }
@@ -1516,7 +1524,7 @@ fn run(self, builder: &Builder<'_>) {
                 .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
         }
 
-        if mode == "run-make" {
+        if mode == "run-make" || mode == "run-coverage" {
             let rust_demangler = builder
                 .ensure(tool::RustDemangler {
                     compiler,
@@ -1703,17 +1711,21 @@ fn run(self, builder: &Builder<'_>) {
                 add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd);
             }
 
-            // Only pass correct values for these flags for the `run-make` suite as it
-            // requires that a C++ compiler was configured which isn't always the case.
-            if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") {
+            if !builder.config.dry_run()
+                && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "run-coverage")
+            {
                 // The llvm/bin directory contains many useful cross-platform
                 // tools. Pass the path to run-make tests so they can use them.
+                // (The run-coverage tests also need these tools to process
+                // coverage reports.)
                 let llvm_bin_path = llvm_config
                     .parent()
                     .expect("Expected llvm-config to be contained in directory");
                 assert!(llvm_bin_path.is_dir());
                 cmd.arg("--llvm-bin-dir").arg(llvm_bin_path);
+            }
 
+            if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") {
                 // If LLD is available, add it to the PATH
                 if builder.config.lld_enabled {
                     let lld_install_root =

src/tools/compiletest/src/common.rs

@@ -66,6 +66,7 @@ pub enum Mode {
         JsDocTest => "js-doc-test",
         MirOpt => "mir-opt",
         Assembly => "assembly",
+        RunCoverage => "run-coverage",
     }
 }
 
@@ -626,6 +627,7 @@ pub fn expected_output_path(
     UI_STDERR_64,
     UI_STDERR_32,
     UI_STDERR_16,
+    UI_COVERAGE,
 ];
 pub const UI_STDERR: &str = "stderr";
 pub const UI_STDOUT: &str = "stdout";
@@ -635,6 +637,7 @@ pub fn expected_output_path(
 pub const UI_STDERR_64: &str = "64bit.stderr";
 pub const UI_STDERR_32: &str = "32bit.stderr";
 pub const UI_STDERR_16: &str = "16bit.stderr";
+pub const UI_COVERAGE: &str = "coverage";
 
 /// Absolute path to the directory where all output for all tests in the given
 /// `relative_dir` group should reside. Example:

src/tools/compiletest/src/header.rs

@@ -161,7 +161,7 @@ pub struct TestProps {
     // customized normalization rules
     pub normalize_stdout: Vec<(String, String)>,
     pub normalize_stderr: Vec<(String, String)>,
-    pub failure_status: i32,
+    pub failure_status: Option<i32>,
     // For UI tests, allows compiler to exit with arbitrary failure status
     pub dont_check_failure_status: bool,
     // Whether or not `rustfix` should apply the `CodeSuggestion`s of this test and compile the
@@ -257,7 +257,7 @@ pub fn new() -> Self {
             check_test_line_numbers_match: false,
             normalize_stdout: vec![],
             normalize_stderr: vec![],
-            failure_status: -1,
+            failure_status: None,
             dont_check_failure_status: false,
             run_rustfix: false,
             rustfix_only_machine_applicable: false,
@@ -428,7 +428,7 @@ fn load_from(&mut self, testfile: &Path, cfg: Option<&str>, config: &Config) {
                     .parse_name_value_directive(ln, FAILURE_STATUS)
                     .and_then(|code| code.trim().parse::<i32>().ok())
                 {
-                    self.failure_status = code;
+                    self.failure_status = Some(code);
                 }
 
                 config.set_name_directive(
@@ -491,11 +491,8 @@ fn load_from(&mut self, testfile: &Path, cfg: Option<&str>, config: &Config) {
             });
         }
 
-        if self.failure_status == -1 {
-            self.failure_status = 1;
-        }
         if self.should_ice {
-            self.failure_status = 101;
+            self.failure_status = Some(101);
         }
 
         if config.mode == Mode::Incremental {
@@ -615,10 +612,25 @@ pub fn line_directive<'line>(
 }
 
 fn iter_header<R: Read>(testfile: &Path, rdr: R, it: &mut dyn FnMut(Option<&str>, &str, usize)) {
+    iter_header_extra(testfile, rdr, &[], it)
+}
+
+fn iter_header_extra(
+    testfile: &Path,
+    rdr: impl Read,
+    extra_directives: &[&str],
+    it: &mut dyn FnMut(Option<&str>, &str, usize),
+) {
     if testfile.is_dir() {
         return;
     }
 
+    // Process any extra directives supplied by the caller (e.g. because they
+    // are implied by the test mode), with a dummy line number of 0.
+    for directive in extra_directives {
+        it(None, directive, 0);
+    }
+
     let comment = if testfile.extension().map(|e| e == "rs") == Some(true) { "//" } else { "#" };
 
     let mut rdr = BufReader::new(rdr);
@@ -897,7 +909,27 @@ pub fn make_test_description<R: Read>(
     let mut ignore_message = None;
     let mut should_fail = false;
 
-    iter_header(path, src, &mut |revision, ln, line_number| {
+    let extra_directives: &[&str] = match config.mode {
+        // The run-coverage tests are treated as having these extra directives,
+        // without needing to specify them manually in every test file.
+        // (Some of the comments below have been copied over from
+        // `tests/run-make/coverage-reports/Makefile`, which no longer exists.)
+        Mode::RunCoverage => {
+            &[
+                "needs-profiler-support",
+                // FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
+                // properly. Since we only have GCC on the CI ignore the test for now.
+                "ignore-windows-gnu",
+                // FIXME(pietroalbini): this test currently does not work on cross-compiled
+                // targets because remote-test is not capable of sending back the *.profraw
+                // files generated by the LLVM instrumentation.
+                "ignore-cross-compile",
+            ]
+        }
+        _ => &[],
+    };
+
+    iter_header_extra(path, src, extra_directives, &mut |revision, ln, line_number| {
         if revision.is_some() && revision != cfg {
             return;
         }

src/tools/compiletest/src/header/needs.rs

@@ -87,7 +87,7 @@ pub(super) fn handle_needs(
         },
         Need {
             name: "needs-profiler-support",
-            condition: std::env::var_os("RUSTC_PROFILER_SUPPORT").is_some(),
+            condition: cache.profiler_support,
             ignore_reason: "ignored when profiler support is disabled",
         },
         Need {
@@ -195,6 +195,7 @@ pub(super) struct CachedNeedsConditions {
     sanitizer_memtag: bool,
     sanitizer_shadow_call_stack: bool,
     sanitizer_safestack: bool,
+    profiler_support: bool,
     xray: bool,
     rust_lld: bool,
     i686_dlltool: bool,
@@ -232,6 +233,7 @@ pub(super) fn load(config: &Config) -> Self {
             sanitizer_memtag: util::MEMTAG_SUPPORTED_TARGETS.contains(target),
             sanitizer_shadow_call_stack: util::SHADOWCALLSTACK_SUPPORTED_TARGETS.contains(target),
             sanitizer_safestack: util::SAFESTACK_SUPPORTED_TARGETS.contains(target),
+            profiler_support: std::env::var_os("RUSTC_PROFILER_SUPPORT").is_some(),
             xray: util::XRAY_SUPPORTED_TARGETS.contains(target),
 
             // For tests using the `needs-rust-lld` directive (e.g. for `-Zgcc-ld=lld`), we need to find

src/tools/compiletest/src/runtest.rs

@@ -6,8 +6,8 @@
 use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{Config, TestPaths};
-use crate::common::{Pretty, RunPassValgrind};
-use crate::common::{UI_RUN_STDERR, UI_RUN_STDOUT};
+use crate::common::{Pretty, RunCoverage, RunPassValgrind};
+use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT};
 use crate::compute_diff::{write_diff, write_filtered_diff};
 use crate::errors::{self, Error, ErrorKind};
 use crate::header::TestProps;
@@ -253,6 +253,7 @@ fn run_revision(&self) {
             MirOpt => self.run_mir_opt_test(),
             Assembly => self.run_assembly_test(),
             JsDocTest => self.run_js_doc_test(),
+            RunCoverage => self.run_coverage_test(),
         }
     }
 
@@ -384,7 +385,7 @@ fn get_output(&self, proc_res: &ProcRes) -> String {
     }
 
     fn check_correct_failure_status(&self, proc_res: &ProcRes) {
-        let expected_status = Some(self.props.failure_status);
+        let expected_status = Some(self.props.failure_status.unwrap_or(1));
         let received_status = proc_res.status.code();
 
         if expected_status != received_status {
@@ -465,6 +466,296 @@ fn run_valgrind_test(&self) {
         }
     }
 
+    fn run_coverage_test(&self) {
+        let should_run = self.run_if_enabled();
+        let proc_res = self.compile_test(should_run, Emit::None);
+
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+        drop(proc_res);
+
+        if let WillExecute::Disabled = should_run {
+            return;
+        }
+
+        let profraw_path = self.output_base_dir().join("default.profraw");
+        let profdata_path = self.output_base_dir().join("default.profdata");
+
+        // Delete any existing profraw/profdata files to rule out unintended
+        // interference between repeated test runs.
+        if profraw_path.exists() {
+            std::fs::remove_file(&profraw_path).unwrap();
+        }
+        if profdata_path.exists() {
+            std::fs::remove_file(&profdata_path).unwrap();
+        }
+
+        let proc_res = self.exec_compiled_test_general(
+            &[("LLVM_PROFILE_FILE", &profraw_path.to_str().unwrap())],
+            false,
+        );
+        if self.props.failure_status.is_some() {
+            self.check_correct_failure_status(&proc_res);
+        } else if !proc_res.status.success() {
+            self.fatal_proc_rec("test run failed!", &proc_res);
+        }
+        drop(proc_res);
+
+        let mut profraw_paths = vec![profraw_path];
+        let mut bin_paths = vec![self.make_exe_name()];
+
+        if self.config.suite == "run-coverage-rustdoc" {
+            self.run_doctests_for_coverage(&mut profraw_paths, &mut bin_paths);
+        }
+
+        // Run `llvm-profdata merge` to index the raw coverage output.
+        let proc_res = self.run_llvm_tool("llvm-profdata", |cmd| {
+            cmd.args(["merge", "--sparse", "--output"]);
+            cmd.arg(&profdata_path);
+            cmd.args(&profraw_paths);
+        });
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("llvm-profdata merge failed!", &proc_res);
+        }
+        drop(proc_res);
+
+        // Run `llvm-cov show` to produce a coverage report in text format.
+        let proc_res = self.run_llvm_tool("llvm-cov", |cmd| {
+            cmd.args(["show", "--format=text", "--show-line-counts-or-regions"]);
+
+            cmd.arg("--Xdemangler");
+            cmd.arg(self.config.rust_demangler_path.as_ref().unwrap());
+
+            cmd.arg("--instr-profile");
+            cmd.arg(&profdata_path);
+
+            for bin in &bin_paths {
+                cmd.arg("--object");
+                cmd.arg(bin);
+            }
+        });
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("llvm-cov show failed!", &proc_res);
+        }
+
+        let kind = UI_COVERAGE;
+
+        let expected_coverage = self.load_expected_output(kind);
+        let normalized_actual_coverage =
+            self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| {
+                self.fatal_proc_rec(&err, &proc_res);
+            });
+
+        let coverage_errors = self.compare_output(
+            kind,
+            &normalized_actual_coverage,
+            &expected_coverage,
+            self.props.compare_output_lines_by_subset,
+        );
+
+        if coverage_errors > 0 {
+            self.fatal_proc_rec(
+                &format!("{} errors occurred comparing coverage output.", coverage_errors),
+                &proc_res,
+            );
+        }
+    }
+
+    /// Run any doctests embedded in this test file, and add any resulting
+    /// `.profraw` files and doctest executables to the given vectors.
+    fn run_doctests_for_coverage(
+        &self,
+        profraw_paths: &mut Vec<PathBuf>,
+        bin_paths: &mut Vec<PathBuf>,
+    ) {
+        // Put .profraw files and doctest executables in dedicated directories,
+        // to make it easier to glob them all later.
+        let profraws_dir = self.output_base_dir().join("doc_profraws");
+        let bins_dir = self.output_base_dir().join("doc_bins");
+
+        // Remove existing directories to prevent cross-run interference.
+        if profraws_dir.try_exists().unwrap() {
+            std::fs::remove_dir_all(&profraws_dir).unwrap();
+        }
+        if bins_dir.try_exists().unwrap() {
+            std::fs::remove_dir_all(&bins_dir).unwrap();
+        }
+
+        let mut rustdoc_cmd =
+            Command::new(self.config.rustdoc_path.as_ref().expect("--rustdoc-path not passed"));
+
+        // In general there will be multiple doctest binaries running, so we
+        // tell the profiler runtime to write their coverage data into separate
+        // profraw files.
+        rustdoc_cmd.env("LLVM_PROFILE_FILE", profraws_dir.join("%p-%m.profraw"));
+
+        rustdoc_cmd.args(["--test", "-Cinstrument-coverage"]);
+
+        // Without this, the doctests complain about not being able to find
+        // their enclosing file's crate for some reason.
+        rustdoc_cmd.args(["--crate-name", "workaround_for_79771"]);
+
+        // Persist the doctest binaries so that `llvm-cov show` can read their
+        // embedded coverage mappings later.
+        rustdoc_cmd.arg("-Zunstable-options");
+        rustdoc_cmd.arg("--persist-doctests");
+        rustdoc_cmd.arg(&bins_dir);
+
+        rustdoc_cmd.arg("-L");
+        rustdoc_cmd.arg(self.aux_output_dir_name());
+
+        rustdoc_cmd.arg(&self.testpaths.file);
+
+        let proc_res = self.compose_and_run_compiler(rustdoc_cmd, None);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("rustdoc --test failed!", &proc_res)
+        }
+
+        fn glob_iter(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
+            let path_str = path.as_ref().to_str().unwrap();
+            let iter = glob(path_str).unwrap();
+            iter.map(Result::unwrap)
+        }
+
+        // Find all profraw files in the profraw directory.
+        for p in glob_iter(profraws_dir.join("*.profraw")) {
+            profraw_paths.push(p);
+        }
+        // Find all executables in the `--persist-doctests` directory, while
+        // avoiding other file types (e.g. `.pdb` on Windows). This doesn't
+        // need to be perfect, as long as it can handle the files actually
+        // produced by `rustdoc --test`.
+        for p in glob_iter(bins_dir.join("**/*")) {
+            let is_bin = p.is_file()
+                && match p.extension() {
+                    None => true,
+                    Some(ext) => ext == OsStr::new("exe"),
+                };
+            if is_bin {
+                bin_paths.push(p);
+            }
+        }
+    }
+
+    fn run_llvm_tool(&self, name: &str, configure_cmd_fn: impl FnOnce(&mut Command)) -> ProcRes {
+        let tool_path = self
+            .config
+            .llvm_bin_dir
+            .as_ref()
+            .expect("this test expects the LLVM bin dir to be available")
+            .join(name);
+
+        let mut cmd = Command::new(tool_path);
+        configure_cmd_fn(&mut cmd);
+
+        let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`"));
+
+        let proc_res = ProcRes {
+            status: output.status,
+            stdout: String::from_utf8(output.stdout).unwrap(),
+            stderr: String::from_utf8(output.stderr).unwrap(),
+            cmdline: format!("{cmd:?}"),
+        };
+        self.dump_output(&proc_res.stdout, &proc_res.stderr);
+
+        proc_res
+    }
+
+    fn normalize_coverage_output(&self, coverage: &str) -> Result<String, String> {
+        let normalized = self.normalize_output(coverage, &[]);
+
+        let mut lines = normalized.lines().collect::<Vec<_>>();
+
+        Self::sort_coverage_file_sections(&mut lines)?;
+        Self::sort_coverage_subviews(&mut lines)?;
+
+        let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
+        Ok(joined_lines)
+    }
+
+    /// Coverage reports can describe multiple source files, separated by
+    /// blank lines. The order of these files is unpredictable (since it
+    /// depends on implementation details), so we need to sort the file
+    /// sections into a consistent order before comparing against a snapshot.
+    fn sort_coverage_file_sections(coverage_lines: &mut Vec<&str>) -> Result<(), String> {
+        // Group the lines into file sections, separated by blank lines.
+        let mut sections = coverage_lines.split(|line| line.is_empty()).collect::<Vec<_>>();
+
+        // The last section should be empty, representing an extra trailing blank line.
+        if !sections.last().is_some_and(|last| last.is_empty()) {
+            return Err("coverage report should end with an extra blank line".to_owned());
+        }
+
+        // Sort the file sections (not including the final empty "section").
+        let except_last = sections.len() - 1;
+        (&mut sections[..except_last]).sort();
+
+        // Join the file sections back into a flat list of lines, with
+        // sections separated by blank lines.
+        let joined = sections.join(&[""] as &[_]);
+        assert_eq!(joined.len(), coverage_lines.len());
+        *coverage_lines = joined;
+
+        Ok(())
+    }
+
+    fn sort_coverage_subviews(coverage_lines: &mut Vec<&str>) -> Result<(), String> {
+        let mut output_lines = Vec::new();
+
+        // We accumulate a list of zero or more "subviews", where each
+        // subview is a list of one or more lines.
+        let mut subviews: Vec<Vec<&str>> = Vec::new();
+
+        fn flush<'a>(subviews: &mut Vec<Vec<&'a str>>, output_lines: &mut Vec<&'a str>) {
+            if subviews.is_empty() {
+                return;
+            }
+
+            // Take and clear the list of accumulated subviews.
+            let mut subviews = std::mem::take(subviews);
+
+            // The last "subview" should be just a boundary line on its own,
+            // so exclude it when sorting the other subviews.
+            let except_last = subviews.len() - 1;
+            (&mut subviews[..except_last]).sort();
+
+            for view in subviews {
+                for line in view {
+                    output_lines.push(line);
+                }
+            }
+        }
+
+        for (line, line_num) in coverage_lines.iter().zip(1..) {
+            if line.starts_with("  ------------------") {
+                // This is a subview boundary line, so start a new subview.
+                subviews.push(vec![line]);
+            } else if line.starts_with("  |") {
+                // Add this line to the current subview.
+                subviews
+                    .last_mut()
+                    .ok_or(format!(
+                        "unexpected subview line outside of a subview on line {line_num}"
+                    ))?
+                    .push(line);
+            } else {
+                // This line is not part of a subview, so sort and print any
+                // accumulated subviews, and then print the line as-is.
+                flush(&mut subviews, &mut output_lines);
+                output_lines.push(line);
+            }
+        }
+
+        flush(&mut subviews, &mut output_lines);
+        assert!(subviews.is_empty());
+
+        assert_eq!(output_lines.len(), coverage_lines.len());
+        *coverage_lines = output_lines;
+
+        Ok(())
+    }
+
     fn run_pretty_test(&self) {
         if self.props.pp_exact.is_some() {
             logv(self.config, "testing for exact pretty-printing".to_owned());
@@ -1598,7 +1889,26 @@ fn document(&self, out_dir: &Path) -> ProcRes {
     }
 
     fn exec_compiled_test(&self) -> ProcRes {
-        let env = &self.props.exec_env;
+        self.exec_compiled_test_general(&[], true)
+    }
+
+    fn exec_compiled_test_general(
+        &self,
+        env_extra: &[(&str, &str)],
+        delete_after_success: bool,
+    ) -> ProcRes {
+        let prepare_env = |cmd: &mut Command| {
+            for key in &self.props.unset_exec_env {
+                cmd.env_remove(key);
+            }
+
+            for (key, val) in &self.props.exec_env {
+                cmd.env(key, val);
+            }
+            for (key, val) in env_extra {
+                cmd.env(key, val);
+            }
+        };
 
         let proc_res = match &*self.config.target {
             // This is pretty similar to below, we're transforming:
@@ -1635,10 +1945,7 @@ fn exec_compiled_test(&self) -> ProcRes {
                     .args(support_libs)
                     .args(args);
 
-                for key in &self.props.unset_exec_env {
-                    test_client.env_remove(key);
-                }
-                test_client.envs(env.clone());
+                prepare_env(&mut test_client);
 
                 self.compose_and_run(
                     test_client,
@@ -1653,10 +1960,7 @@ fn exec_compiled_test(&self) -> ProcRes {
                 let mut wr_run = Command::new("wr-run");
                 wr_run.args(&[&prog]).args(args);
 
-                for key in &self.props.unset_exec_env {
-                    wr_run.env_remove(key);
-                }
-                wr_run.envs(env.clone());
+                prepare_env(&mut wr_run);
 
                 self.compose_and_run(
                     wr_run,
@@ -1671,10 +1975,7 @@ fn exec_compiled_test(&self) -> ProcRes {
                 let mut program = Command::new(&prog);
                 program.args(args).current_dir(&self.output_base_dir());
 
-                for key in &self.props.unset_exec_env {
-                    program.env_remove(key);
-                }
-                program.envs(env.clone());
+                prepare_env(&mut program);
 
                 self.compose_and_run(
                     program,
@@ -1685,7 +1986,7 @@ fn exec_compiled_test(&self) -> ProcRes {
             }
         };
 
-        if proc_res.status.success() {
+        if delete_after_success && proc_res.status.success() {
             // delete the executable after running it to save space.
             // it is ok if the deletion failed.
             let _ = fs::remove_file(self.make_exe_name());
@@ -1812,6 +2113,7 @@ fn build_auxiliary(&self, source_path: &str, aux_dir: &Path) -> bool {
             || self.is_vxworks_pure_static()
             || self.config.target.contains("bpf")
             || !self.config.target_cfg().dynamic_linking
+            || self.config.mode == RunCoverage
         {
             // We primarily compile all auxiliary libraries as dynamic libraries
             // to avoid code size bloat and large binaries as much as possible
@@ -1822,6 +2124,10 @@ fn build_auxiliary(&self, source_path: &str, aux_dir: &Path) -> bool {
             // dynamic libraries so we just go back to building a normal library. Note,
             // however, that for MUSL if the library is built with `force_host` then
             // it's ok to be a dylib as the host should always support dylibs.
+            //
+            // Coverage tests want static linking by default so that coverage
+            // mappings in auxiliary libraries can be merged into the final
+            // executable.
             (false, Some("lib"))
         } else {
             (true, Some("dylib"))
@@ -1999,6 +2305,10 @@ fn make_compile_args(
                     }
                 }
                 DebugInfo => { /* debuginfo tests must be unoptimized */ }
+                RunCoverage => {
+                    // Coverage reports are affected by optimization level, and
+                    // the current snapshots assume no optimization by default.
+                }
                 _ => {
                     rustc.arg("-O");
                 }
@@ -2065,6 +2375,9 @@ fn make_compile_args(
 
                 rustc.arg(dir_opt);
             }
+            RunCoverage => {
+                rustc.arg("-Cinstrument-coverage");
+            }
             RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake
             | CodegenUnits | JsDocTest | Assembly => {
                 // do not use JSON output

tests/run-coverage-rustdoc/doctest.coverage

@@ -1,4 +1,15 @@
-../coverage/doctest.rs:
+$DIR/auxiliary/doctest_crate.rs:
+    1|       |/// A function run only from within doctests
+    2|      3|pub fn fn_run_in_doctests(conditional: usize) {
+    3|      3|    match conditional {
+    4|      1|        1 => assert_eq!(1, 1), // this is run,
+    5|      1|        2 => assert_eq!(1, 1), // this,
+    6|      1|        3 => assert_eq!(1, 1), // and this too
+    7|      0|        _ => assert_eq!(1, 2), // however this is not
+    8|       |    }
+    9|      3|}
+
+$DIR/doctest.rs:
     1|       |//! This test ensures that code from doctests is properly re-mapped.
     2|       |//! See <https://github.com/rust-lang/rust/issues/79417> for more info.
     3|       |//!
@@ -67,7 +78,7 @@
    63|       |//!     doctest_main()
    64|       |//! }
    65|       |//! ```
-   66|       |
+   66|       |// aux-build:doctest_crate.rs
    67|       |/// doctest attached to fn testing external code:
    68|       |/// ```
    69|      1|/// extern crate doctest_crate;
@@ -102,14 +113,3 @@
    98|       |// what affect it might have on diagnostic messages from the compiler, and whether anyone would care
    99|       |// if the indentation changed. I don't know if there is a more viable solution.
 
-../coverage/lib/doctest_crate.rs:
-    1|       |/// A function run only from within doctests
-    2|      3|pub fn fn_run_in_doctests(conditional: usize) {
-    3|      3|    match conditional {
-    4|      1|        1 => assert_eq!(1, 1), // this is run,
-    5|      1|        2 => assert_eq!(1, 1), // this,
-    6|      1|        3 => assert_eq!(1, 1), // and this too
-    7|      0|        _ => assert_eq!(1, 2), // however this is not
-    8|       |    }
-    9|      3|}
-

tests/run-coverage-rustdoc/doctest.rs

@@ -63,7 +63,7 @@
 //!     doctest_main()
 //! }
 //! ```
-
+// aux-build:doctest_crate.rs
 /// doctest attached to fn testing external code:
 /// ```
 /// extern crate doctest_crate;

tests/run-coverage/assert.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-101
+    2|       |// failure-status: 101
     3|       |
     4|      4|fn might_fail_assert(one_plus_one: u32) {
     5|      4|    println!("does 1 + 1 = {}?", one_plus_one);

tests/run-coverage/assert.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-101
+// failure-status: 101
 
 fn might_fail_assert(one_plus_one: u32) {
     println!("does 1 + 1 = {}?", one_plus_one);

tests/run-coverage/auxiliary/used_crate.rs

@@ -1,6 +1,6 @@
 #![allow(unused_assignments, unused_variables)]
-// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
-use std::fmt::Debug;
+// compile-flags: -C opt-level=3
+use std::fmt::Debug; // ^^ validates coverage now works with optimizations
 
 pub fn used_function() {
     // Initialize test constants in a way that cannot be determined at compile time, to ensure

tests/run-coverage/auxiliary/used_inline_crate.rs

@@ -1,7 +1,7 @@
 #![allow(unused_assignments, unused_variables)]
 
-// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
-
+// compile-flags: -C opt-level=3
+// ^^ validates coverage now works with optimizations
 use std::fmt::Debug;
 
 pub fn used_function() {

tests/run-coverage/closure.coverage

@@ -1,6 +1,6 @@
     1|       |#![allow(unused_assignments, unused_variables)]
-    2|       |// compile-flags: -C opt-level=2 # fix described in rustc_middle/mir/mono.rs
-    3|      1|fn main() {
+    2|       |// compile-flags: -C opt-level=2
+    3|      1|fn main() { // ^^ fix described in rustc_middle/mir/mono.rs
     4|      1|    // Initialize test constants in a way that cannot be determined at compile time, to ensure
     5|      1|    // rustc and LLVM cannot optimize out statements (or coverage counters) downstream from
     6|      1|    // dependent conditions.

tests/run-coverage/closure.rs

@@ -1,6 +1,6 @@
 #![allow(unused_assignments, unused_variables)]
-// compile-flags: -C opt-level=2 # fix described in rustc_middle/mir/mono.rs
-fn main() {
+// compile-flags: -C opt-level=2
+fn main() { // ^^ fix described in rustc_middle/mir/mono.rs
     // Initialize test constants in a way that cannot be determined at compile time, to ensure
     // rustc and LLVM cannot optimize out statements (or coverage counters) downstream from
     // dependent conditions.

tests/run-coverage/drop_trait.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-1
+    2|       |// failure-status: 1
     3|       |
     4|       |struct Firework {
     5|       |    strength: i32,

tests/run-coverage/drop_trait.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-1
+// failure-status: 1
 
 struct Firework {
     strength: i32,

tests/run-coverage/generics.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-1
+    2|       |// failure-status: 1
     3|       |
     4|       |struct Firework<T> where T: Copy + std::fmt::Display {
     5|       |    strength: T,

tests/run-coverage/generics.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-1
+// failure-status: 1
 
 struct Firework<T> where T: Copy + std::fmt::Display {
     strength: T,

tests/run-coverage/issue-84561.coverage

@@ -1,6 +1,6 @@
     1|       |// This demonstrated Issue #84561: function-like macros produce unintuitive coverage results.
     2|       |
-    3|       |// expect-exit-status-101
+    3|       |// failure-status: 101
     4|     21|#[derive(PartialEq, Eq)]
     5|       |struct Foo(u32);
     6|      1|fn test3() {

tests/run-coverage/issue-84561.rs

@@ -1,6 +1,6 @@
 // This demonstrated Issue #84561: function-like macros produce unintuitive coverage results.
 
-// expect-exit-status-101
+// failure-status: 101
 #[derive(PartialEq, Eq)]
 struct Foo(u32);
 fn test3() {

tests/run-coverage/issue-85461.coverage

@@ -1,16 +1,4 @@
-../coverage/issue-85461.rs:
-    1|       |// Regression test for #85461: MSVC sometimes fail to link with dead code and #[inline(always)]
-    2|       |
-    3|       |extern crate inline_always_with_dead_code;
-    4|       |
-    5|       |use inline_always_with_dead_code::{bar, baz};
-    6|       |
-    7|      1|fn main() {
-    8|      1|    bar::call_me();
-    9|      1|    baz::call_me();
-   10|      1|}
-
-../coverage/lib/inline_always_with_dead_code.rs:
+$DIR/auxiliary/inline_always_with_dead_code.rs:
     1|       |// compile-flags: -Cinstrument-coverage -Ccodegen-units=4 -Copt-level=0
     2|       |
     3|       |#![allow(dead_code)]
@@ -34,3 +22,15 @@
    21|      1|    }
    22|       |}
 
+$DIR/issue-85461.rs:
+    1|       |// Regression test for #85461: MSVC sometimes fail to link with dead code and #[inline(always)]
+    2|       |// aux-build:inline_always_with_dead_code.rs
+    3|       |extern crate inline_always_with_dead_code;
+    4|       |
+    5|       |use inline_always_with_dead_code::{bar, baz};
+    6|       |
+    7|      1|fn main() {
+    8|      1|    bar::call_me();
+    9|      1|    baz::call_me();
+   10|      1|}
+

tests/run-coverage/issue-85461.rs

@@ -1,5 +1,5 @@
 // Regression test for #85461: MSVC sometimes fail to link with dead code and #[inline(always)]
-
+// aux-build:inline_always_with_dead_code.rs
 extern crate inline_always_with_dead_code;
 
 use inline_always_with_dead_code::{bar, baz};

tests/run-coverage/overflow.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-101
+    2|       |// failure-status: 101
     3|       |
     4|      4|fn might_overflow(to_add: u32) -> u32 {
     5|      4|    if to_add > 5 {

tests/run-coverage/overflow.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-101
+// failure-status: 101
 
 fn might_overflow(to_add: u32) -> u32 {
     if to_add > 5 {

tests/run-coverage/panic_unwind.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-101
+    2|       |// failure-status: 101
     3|       |
     4|      4|fn might_panic(should_panic: bool) {
     5|      4|    if should_panic {

tests/run-coverage/panic_unwind.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-101
+// failure-status: 101
 
 fn might_panic(should_panic: bool) {
     if should_panic {

tests/run-coverage/try_error_result.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-1
+    2|       |// failure-status: 1
     3|       |
     4|      6|fn call(return_error: bool) -> Result<(),()> {
     5|      6|    if return_error {

tests/run-coverage/try_error_result.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-1
+// failure-status: 1
 
 fn call(return_error: bool) -> Result<(),()> {
     if return_error {

tests/run-coverage/unused_mod.coverage

@@ -0,0 +1,13 @@
+$DIR/auxiliary/unused_mod_helper.rs:
+    1|      0|pub fn never_called_function() {
+    2|      0|    println!("I am never called");
+    3|      0|}
+
+$DIR/unused_mod.rs:
+    1|       |#[path = "auxiliary/unused_mod_helper.rs"]
+    2|       |mod unused_module;
+    3|       |
+    4|      1|fn main() {
+    5|      1|    println!("hello world!");
+    6|      1|}
+

tests/run-coverage/unused_mod.rs

@@ -1,4 +1,4 @@
-#[path = "lib/unused_mod_helper.rs"]
+#[path = "auxiliary/unused_mod_helper.rs"]
 mod unused_module;
 
 fn main() {

tests/run-coverage/uses_crate.coverage

@@ -1,6 +1,7 @@
+$DIR/auxiliary/used_crate.rs:
     1|       |#![allow(unused_assignments, unused_variables)]
-    2|       |// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
-    3|       |use std::fmt::Debug;
+    2|       |// compile-flags: -C opt-level=3
+    3|       |use std::fmt::Debug; // ^^ validates coverage now works with optimizations
     4|       |
     5|      1|pub fn used_function() {
     6|      1|    // Initialize test constants in a way that cannot be determined at compile time, to ensure
@@ -146,3 +147,24 @@
    99|       |// functions" list, which would then omit coverage results for
   100|       |// `unused_generic_function<T>()`, below.
 
+$DIR/uses_crate.rs:
+    1|       |// FIXME #110395
+    2|       |// ignore-linux
+    3|       |
+    4|       |// Validates coverage now works with optimizations
+    5|       |// compile-flags: -C opt-level=3
+    6|       |
+    7|       |#![allow(unused_assignments, unused_variables)]
+    8|       |
+    9|       |// aux-build:used_crate.rs
+   10|       |extern crate used_crate;
+   11|       |
+   12|      1|fn main() {
+   13|      1|    used_crate::used_function();
+   14|      1|    let some_vec = vec![1, 2, 3, 4];
+   15|      1|    used_crate::used_only_from_bin_crate_generic_function(&some_vec);
+   16|      1|    used_crate::used_only_from_bin_crate_generic_function("used from bin uses_crate.rs");
+   17|      1|    used_crate::used_from_bin_crate_and_lib_crate_generic_function(some_vec);
+   18|      1|    used_crate::used_with_same_type_from_bin_crate_and_lib_crate_generic_function("interesting?");
+   19|      1|}
+

tests/run-coverage/uses_crate.rs

@@ -1,8 +1,12 @@
 // FIXME #110395
-// ignore-llvm-cov-show-diffs
+// ignore-linux
+
+// Validates coverage now works with optimizations
+// compile-flags: -C opt-level=3
 
 #![allow(unused_assignments, unused_variables)]
-// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
+
+// aux-build:used_crate.rs
 extern crate used_crate;
 
 fn main() {

tests/run-coverage/uses_inline_crate.coverage

@@ -1,7 +1,8 @@
+$DIR/auxiliary/used_inline_crate.rs:
     1|       |#![allow(unused_assignments, unused_variables)]
     2|       |
-    3|       |// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
-    4|       |
+    3|       |// compile-flags: -C opt-level=3
+    4|       |// ^^ validates coverage now works with optimizations
     5|       |use std::fmt::Debug;
     6|       |
     7|      1|pub fn used_function() {
@@ -137,3 +138,27 @@
    89|      2|    used_only_from_this_lib_crate_generic_function("used ONLY from library used_crate.rs");
    90|      2|}
 
+$DIR/uses_inline_crate.rs:
+    1|       |// FIXME #110395
+    2|       |// ignore-linux
+    3|       |
+    4|       |// Validates coverage now works with optimizations
+    5|       |// compile-flags: -C opt-level=3
+    6|       |
+    7|       |#![allow(unused_assignments, unused_variables)]
+    8|       |
+    9|       |// aux-build:used_inline_crate.rs
+   10|       |extern crate used_inline_crate;
+   11|       |
+   12|      1|fn main() {
+   13|      1|    used_inline_crate::used_function();
+   14|      1|    used_inline_crate::used_inline_function();
+   15|      1|    let some_vec = vec![1, 2, 3, 4];
+   16|      1|    used_inline_crate::used_only_from_bin_crate_generic_function(&some_vec);
+   17|      1|    used_inline_crate::used_only_from_bin_crate_generic_function("used from bin uses_crate.rs");
+   18|      1|    used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function(some_vec);
+   19|      1|    used_inline_crate::used_with_same_type_from_bin_crate_and_lib_crate_generic_function(
+   20|      1|        "interesting?",
+   21|      1|    );
+   22|      1|}
+

tests/run-coverage/uses_inline_crate.rs

@@ -1,10 +1,12 @@
 // FIXME #110395
-// ignore-llvm-cov-show-diffs
+// ignore-linux
+
+// Validates coverage now works with optimizations
+// compile-flags: -C opt-level=3
 
 #![allow(unused_assignments, unused_variables)]
 
-// compile-flags: -C opt-level=3 # validates coverage now works with optimizations
-
+// aux-build:used_inline_crate.rs
 extern crate used_inline_crate;
 
 fn main() {

tests/run-coverage/while_early_ret.coverage

@@ -1,5 +1,5 @@
     1|       |#![allow(unused_assignments)]
-    2|       |// expect-exit-status-1
+    2|       |// failure-status: 1
     3|       |
     4|      1|fn main() -> Result<(),u8> {
     5|      1|    let mut countdown = 10;

tests/run-coverage/while_early_ret.rs

@@ -1,5 +1,5 @@
 #![allow(unused_assignments)]
-// expect-exit-status-1
+// failure-status: 1
 
 fn main() -> Result<(),u8> {
     let mut countdown = 10;