diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 7b396dde91b..0184fec8e3e 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -84,6 +84,7 @@ pub fn insert(&self, symbol: Symbol, span: Span) {
 
 /// Construct a diagnostic for a language feature error due to the given `span`.
 /// The `feature`'s `Symbol` is the one you used in `active.rs` and `rustc_span::symbols`.
+#[track_caller]
 pub fn feature_err(
     sess: &ParseSess,
     feature: Symbol,
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index 2b72d6c48eb..b495ed65268 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -1753,10 +1753,6 @@ fn run(self, builder: &Builder<'_>) {
             cmd.arg("--git-hash");
         }
 
-        if let Some(commit) = builder.config.download_rustc_commit() {
-            cmd.env("FAKE_DOWNLOAD_RUSTC_PREFIX", format!("/rustc/{commit}"));
-        }
-
         builder.ci_env.force_coloring_in_ci(&mut cmd);
 
         builder.info(&format!(
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 5bc4d164265..c2e18ce2547 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -1939,6 +1939,17 @@ fn make_compile_args(
         // Use a single thread for efficiency and a deterministic error message order
         rustc.arg("-Zthreads=1");
 
+        // Hide libstd sources from ui tests to make sure we generate the stderr
+        // output that users will see.
+        // Without this, we may be producing good diagnostics in-tree but users
+        // will not see half the information.
+        //
+        // This also has the benefit of more effectively normalizing output between different
+        // compilers, so that we don't have to know the `/rustc/$sha` output to normalize after the
+        // fact.
+        rustc.arg("-Zsimulate-remapped-rust-src-base=/rustc/FAKE_PREFIX");
+        rustc.arg("-Ztranslate-remapped-path-to-local-path=no");
+
         // Optionally prevent default --sysroot if specified in test compile-flags.
         if !self.props.compile_flags.iter().any(|flag| flag.starts_with("--sysroot")) {
             // In stage 0, make sure we use `stage0-sysroot` instead of the bootstrap sysroot.
@@ -2014,13 +2025,6 @@ fn make_compile_args(
                 rustc.arg("-Ccodegen-units=1");
                 // Hide line numbers to reduce churn
                 rustc.arg("-Zui-testing");
-                // Hide libstd sources from ui tests to make sure we generate the stderr
-                // output that users will see.
-                // Without this, we may be producing good diagnostics in-tree but users
-                // will not see half the information.
-                rustc.arg("-Zsimulate-remapped-rust-src-base=/rustc/FAKE_PREFIX");
-                rustc.arg("-Ztranslate-remapped-path-to-local-path=no");
-
                 rustc.arg("-Zdeduplicate-diagnostics=no");
                 // FIXME: use this for other modes too, for perf?
                 rustc.arg("-Cstrip=debuginfo");
@@ -3729,28 +3733,13 @@ fn normalize_output(&self, output: &str, custom_rules: &[(String, String)]) -> S
             normalize_path(&remapped_parent_dir, "$DIR");
         }
 
-        let source_bases = &[
-            // Source base on the current filesystem (calculated as parent of `tests/$suite`):
-            Some(self.config.src_base.parent().unwrap().parent().unwrap().into()),
-            // Source base on the sysroot (from the src components downloaded by `download-rustc`):
-            Some(self.config.sysroot_base.join("lib").join("rustlib").join("src").join("rust")),
-            // Virtual `/rustc/$sha` remapped paths (if `remap-debuginfo` is enabled):
-            option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(PathBuf::from),
-            // Virtual `/rustc/$sha` coming from download-rustc:
-            std::env::var_os("FAKE_DOWNLOAD_RUSTC_PREFIX").map(PathBuf::from),
-            // Tests using -Zsimulate-remapped-rust-src-base should use this fake path
-            Some("/rustc/FAKE_PREFIX".into()),
-        ];
-        for base_dir in source_bases {
-            if let Some(base_dir) = base_dir {
-                // Paths into the libstd/libcore
-                normalize_path(&base_dir.join("library"), "$SRC_DIR");
-                // `ui-fulldeps` tests can show paths to the compiler source when testing macros from
-                // `rustc_macros`
-                // eg. /home/user/rust/compiler
-                normalize_path(&base_dir.join("compiler"), "$COMPILER_DIR");
-            }
-        }
+        let base_dir = Path::new("/rustc/FAKE_PREFIX");
+        // Paths into the libstd/libcore
+        normalize_path(&base_dir.join("library"), "$SRC_DIR");
+        // `ui-fulldeps` tests can show paths to the compiler source when testing macros from
+        // `rustc_macros`
+        // eg. /home/user/rust/compiler
+        normalize_path(&base_dir.join("compiler"), "$COMPILER_DIR");
 
         // Paths into the build directory
         let test_build_dir = &self.config.build_base;
diff --git a/tests/ui/track-diagnostics/track6.stderr b/tests/ui/track-diagnostics/track6.stderr
index 89438aea9ad..583b02555b4 100644
--- a/tests/ui/track-diagnostics/track6.stderr
+++ b/tests/ui/track-diagnostics/track6.stderr
@@ -3,7 +3,7 @@ error[E0658]: specialization is unstable
    |
 LL |     default fn bar() {}
    |     ^^^^^^^^^^^^^^^^^^^
--Ztrack-diagnostics: created at $COMPILER_DIR/rustc_session/src/parse.rs:LL:CC
+-Ztrack-diagnostics: created at compiler/rustc_ast_passes/src/feature_gate.rs:LL:CC
    |
    = note: see issue #31844 <https://github.com/rust-lang/rust/issues/31844> for more information
    = help: add `#![feature(specialization)]` to the crate attributes to enable