diff --git a/src/tools/run-make-support/src/command.rs b/src/tools/run-make-support/src/command.rs
index 47376c401bb..6f9ff4b9868 100644
--- a/src/tools/run-make-support/src/command.rs
+++ b/src/tools/run-make-support/src/command.rs
@@ -216,7 +216,7 @@ pub fn assert_stderr_contains<S: AsRef<str>>(&self, expected: S) -> &Self {
     /// Checks that `stderr` does not contain `unexpected`.
     #[track_caller]
     pub fn assert_stderr_not_contains<S: AsRef<str>>(&self, unexpected: S) -> &Self {
-        assert_not_contains(&self.stdout_utf8(), unexpected);
+        assert_not_contains(&self.stderr_utf8(), unexpected);
         self
     }
 
diff --git a/src/tools/tidy/src/allowed_run_make_makefiles.txt b/src/tools/tidy/src/allowed_run_make_makefiles.txt
index 2a1b7500e5b..beff9de2371 100644
--- a/src/tools/tidy/src/allowed_run_make_makefiles.txt
+++ b/src/tools/tidy/src/allowed_run_make_makefiles.txt
@@ -17,7 +17,6 @@ run-make/foreign-double-unwind/Makefile
 run-make/foreign-exceptions/Makefile
 run-make/foreign-rust-exceptions/Makefile
 run-make/incr-add-rust-src-component/Makefile
-run-make/issue-35164/Makefile
 run-make/issue-36710/Makefile
 run-make/issue-69368/Makefile
 run-make/issue-84395-lto-embed-bitcode/Makefile
diff --git a/tests/run-make/issue-35164/Makefile b/tests/run-make/issue-35164/Makefile
deleted file mode 100644
index 38aa6f1265f..00000000000
--- a/tests/run-make/issue-35164/Makefile
+++ /dev/null
@@ -1,4 +0,0 @@
-include ../tools.mk
-
-all:
-	$(RUSTC) main.rs --error-format json 2>&1 | $(CGREP) -e '"byte_start":23\b' '"byte_end":29\b'
diff --git a/tests/run-make/issue-35164/main.rs b/tests/run-make/json-error-no-offset/main.rs
similarity index 100%
rename from tests/run-make/issue-35164/main.rs
rename to tests/run-make/json-error-no-offset/main.rs
diff --git a/tests/run-make/json-error-no-offset/rmake.rs b/tests/run-make/json-error-no-offset/rmake.rs
new file mode 100644
index 00000000000..629d9c4c16e
--- /dev/null
+++ b/tests/run-make/json-error-no-offset/rmake.rs
@@ -0,0 +1,15 @@
+// The byte positions in json format error logging used to have a small, difficult
+// to predict offset. This was changed to be the top of the file every time in #42973,
+// and this test checks that the measurements appearing in the standard error are correct.
+// See https://github.com/rust-lang/rust/issues/35164
+
+use run_make_support::rustc;
+
+fn main() {
+    rustc()
+        .input("main.rs")
+        .error_format("json")
+        .run()
+        .assert_stderr_contains(r#""byte_start":23"#)
+        .assert_stderr_contains(r#""byte_end":29"#);
+}
diff --git a/tests/run-make/issue-35164/submodule/mod.rs b/tests/run-make/json-error-no-offset/submodule/mod.rs
similarity index 100%
rename from tests/run-make/issue-35164/submodule/mod.rs
rename to tests/run-make/json-error-no-offset/submodule/mod.rs