Auto merge of #122898 - jieyouxu:tidy-no-new-makefiles, r=albertlarsan68

Add tidy check to error on new `Makefile`s in `tests/run-make`

We would like to strongly encourage new `run-make` tests to be written in Rust and not add any new `Makefile`-based `run-make` tests.

src/tools/tidy/src/allowed_run_make_makefiles.txt

@@ -0,0 +1,337 @@
+run-make/alloc-no-oom-handling/Makefile
+run-make/alloc-no-rc/Makefile
+run-make/alloc-no-sync/Makefile
+run-make/allocator-shim-circular-deps/Makefile
+run-make/allow-non-lint-warnings-cmdline/Makefile
+run-make/allow-warnings-cmdline-stability/Makefile
+run-make/archive-duplicate-names/Makefile
+run-make/arguments-non-c-like-enum/Makefile
+run-make/atomic-lock-free/Makefile
+run-make/bare-outfile/Makefile
+run-make/branch-protection-check-IBT/Makefile
+run-make/c-dynamic-dylib/Makefile
+run-make/c-dynamic-rlib/Makefile
+run-make/c-link-to-rust-dylib/Makefile
+run-make/c-link-to-rust-staticlib/Makefile
+run-make/c-link-to-rust-va-list-fn/Makefile
+run-make/c-static-dylib/Makefile
+run-make/c-static-rlib/Makefile
+run-make/c-unwind-abi-catch-lib-panic/Makefile
+run-make/c-unwind-abi-catch-panic/Makefile
+run-make/cat-and-grep-sanity-check/Makefile
+run-make/cdylib-dylib-linkage/Makefile
+run-make/cdylib-fewer-symbols/Makefile
+run-make/cdylib/Makefile
+run-make/codegen-options-parsing/Makefile
+run-make/comment-section/Makefile
+run-make/compile-stdin/Makefile
+run-make/compiler-lookup-paths-2/Makefile
+run-make/compiler-lookup-paths/Makefile
+run-make/compiler-rt-works-on-mingw/Makefile
+run-make/compressed-debuginfo/Makefile
+run-make/const-prop-lint/Makefile
+run-make/const_fn_mir/Makefile
+run-make/core-no-fp-fmt-parse/Makefile
+run-make/core-no-oom-handling/Makefile
+run-make/crate-data-smoke/Makefile
+run-make/crate-hash-rustc-version/Makefile
+run-make/crate-name-priority/Makefile
+run-make/cross-lang-lto-clang/Makefile
+run-make/cross-lang-lto-pgo-smoketest/Makefile
+run-make/cross-lang-lto-upstream-rlibs/Makefile
+run-make/cross-lang-lto/Makefile
+run-make/debug-assertions/Makefile
+run-make/debugger-visualizer-dep-info/Makefile
+run-make/dep-graph/Makefile
+run-make/dep-info-doesnt-run-much/Makefile
+run-make/dep-info-spaces/Makefile
+run-make/dep-info/Makefile
+run-make/doctests-keep-binaries/Makefile
+run-make/doctests-runtool/Makefile
+run-make/dump-ice-to-disk/Makefile
+run-make/dump-mono-stats/Makefile
+run-make/duplicate-output-flavors/Makefile
+run-make/dylib-chain/Makefile
+run-make/emit-named-files/Makefile
+run-make/emit-path-unhashed/Makefile
+run-make/emit-shared-files/Makefile
+run-make/emit-stack-sizes/Makefile
+run-make/emit-to-stdout/Makefile
+run-make/emit/Makefile
+run-make/env-dep-info/Makefile
+run-make/error-found-staticlib-instead-crate/Makefile
+run-make/error-writing-dependencies/Makefile
+run-make/exit-code/Makefile
+run-make/export-executable-symbols/Makefile
+run-make/extern-diff-internal-name/Makefile
+run-make/extern-flag-disambiguates/Makefile
+run-make/extern-flag-fun/Makefile
+run-make/extern-flag-pathless/Makefile
+run-make/extern-flag-rename-transitive/Makefile
+run-make/extern-fn-explicit-align/Makefile
+run-make/extern-fn-generic/Makefile
+run-make/extern-fn-mangle/Makefile
+run-make/extern-fn-reachable/Makefile
+run-make/extern-fn-struct-passing-abi/Makefile
+run-make/extern-fn-with-extern-types/Makefile
+run-make/extern-fn-with-packed-struct/Makefile
+run-make/extern-fn-with-union/Makefile
+run-make/extern-multiple-copies/Makefile
+run-make/extern-multiple-copies2/Makefile
+run-make/extern-overrides-distribution/Makefile
+run-make/extra-filename-with-temp-outputs/Makefile
+run-make/fmt-write-bloat/Makefile
+run-make/forced-unwind-terminate-pof/Makefile
+run-make/foreign-double-unwind/Makefile
+run-make/foreign-exceptions/Makefile
+run-make/foreign-rust-exceptions/Makefile
+run-make/fpic/Makefile
+run-make/glibc-staticlib-args/Makefile
+run-make/hir-tree/Makefile
+run-make/inaccessible-temp-dir/Makefile
+run-make/include_bytes_deps/Makefile
+run-make/incr-add-rust-src-component/Makefile
+run-make/incr-foreign-head-span/Makefile
+run-make/incr-prev-body-beyond-eof/Makefile
+run-make/incremental-debugger-visualizer/Makefile
+run-make/incremental-session-fail/Makefile
+run-make/inline-always-many-cgu/Makefile
+run-make/interdependent-c-libraries/Makefile
+run-make/intrinsic-unreachable/Makefile
+run-make/invalid-library/Makefile
+run-make/invalid-so/Makefile
+run-make/invalid-staticlib/Makefile
+run-make/issue-107094/Makefile
+run-make/issue-10971-temps-dir/Makefile
+run-make/issue-109934-lto-debuginfo/Makefile
+run-make/issue-11908/Makefile
+run-make/issue-14500/Makefile
+run-make/issue-14698/Makefile
+run-make/issue-15460/Makefile
+run-make/issue-18943/Makefile
+run-make/issue-20626/Makefile
+run-make/issue-22131/Makefile
+run-make/issue-24445/Makefile
+run-make/issue-25581/Makefile
+run-make/issue-26006/Makefile
+run-make/issue-26092/Makefile
+run-make/issue-28595/Makefile
+run-make/issue-28766/Makefile
+run-make/issue-30063/Makefile
+run-make/issue-33329/Makefile
+run-make/issue-35164/Makefile
+run-make/issue-36710/Makefile
+run-make/issue-37839/Makefile
+run-make/issue-37893/Makefile
+run-make/issue-38237/Makefile
+run-make/issue-40535/Makefile
+run-make/issue-46239/Makefile
+run-make/issue-47384/Makefile
+run-make/issue-47551/Makefile
+run-make/issue-51671/Makefile
+run-make/issue-53964/Makefile
+run-make/issue-64153/Makefile
+run-make/issue-68794-textrel-on-minimal-lib/Makefile
+run-make/issue-69368/Makefile
+run-make/issue-7349/Makefile
+run-make/issue-83045/Makefile
+run-make/issue-83112-incr-test-moved-file/Makefile
+run-make/issue-84395-lto-embed-bitcode/Makefile
+run-make/issue-85019-moved-src-dir/Makefile
+run-make/issue-85401-static-mir/Makefile
+run-make/issue-85441/Makefile
+run-make/issue-88756-default-output/Makefile
+run-make/issue-97463-abi-param-passing/Makefile
+run-make/issue64319/Makefile
+run-make/jobserver-error/Makefile
+run-make/libs-through-symlinks/Makefile
+run-make/libtest-json/Makefile
+run-make/libtest-junit/Makefile
+run-make/libtest-padding/Makefile
+run-make/libtest-thread-limit/Makefile
+run-make/link-arg/Makefile
+run-make/link-args-order/Makefile
+run-make/link-cfg/Makefile
+run-make/link-dedup/Makefile
+run-make/link-framework/Makefile
+run-make/link-path-order/Makefile
+run-make/linkage-attr-on-static/Makefile
+run-make/llvm-ident/Makefile
+run-make/llvm-outputs/Makefile
+run-make/long-linker-command-lines-cmd-exe/Makefile
+run-make/long-linker-command-lines/Makefile
+run-make/longjmp-across-rust/Makefile
+run-make/ls-metadata/Makefile
+run-make/lto-dylib-dep/Makefile
+run-make/lto-empty/Makefile
+run-make/lto-linkage-used-attr/Makefile
+run-make/lto-no-link-whole-rlib/Makefile
+run-make/lto-readonly-lib/Makefile
+run-make/lto-smoke-c/Makefile
+run-make/lto-smoke/Makefile
+run-make/macos-deployment-target/Makefile
+run-make/macos-fat-archive/Makefile
+run-make/manual-crate-name/Makefile
+run-make/manual-link/Makefile
+run-make/many-crates-but-no-match/Makefile
+run-make/metadata-dep-info/Makefile
+run-make/metadata-flag-frobs-symbols/Makefile
+run-make/min-global-align/Makefile
+run-make/mingw-export-call-convention/Makefile
+run-make/mismatching-target-triples/Makefile
+run-make/missing-crate-dependency/Makefile
+run-make/mixing-deps/Makefile
+run-make/mixing-formats/Makefile
+run-make/mixing-libs/Makefile
+run-make/msvc-opt-minsize/Makefile
+run-make/multiple-emits/Makefile
+run-make/native-link-modifier-bundle/Makefile
+run-make/native-link-modifier-verbatim-linker/Makefile
+run-make/native-link-modifier-verbatim-rustc/Makefile
+run-make/native-link-modifier-whole-archive/Makefile
+run-make/no-alloc-shim/Makefile
+run-make/no-builtins-attribute/Makefile
+run-make/no-builtins-lto/Makefile
+run-make/no-cdylib-as-rdylib/Makefile
+run-make/no-duplicate-libs/Makefile
+run-make/no-input-file/Makefile
+run-make/no-intermediate-extras/Makefile
+run-make/obey-crate-type-flag/Makefile
+run-make/optimization-remarks-dir-pgo/Makefile
+run-make/optimization-remarks-dir/Makefile
+run-make/output-filename-conflicts-with-directory/Makefile
+run-make/output-filename-overwrites-input/Makefile
+run-make/output-type-permutations/Makefile
+run-make/output-with-hyphens/Makefile
+run-make/override-aliased-flags/Makefile
+run-make/overwrite-input/Makefile
+run-make/panic-abort-eh_frame/Makefile
+run-make/panic-impl-transitive/Makefile
+run-make/pass-linker-flags-flavor/Makefile
+run-make/pass-linker-flags-from-dep/Makefile
+run-make/pass-linker-flags/Makefile
+run-make/pass-non-c-like-enum-to-c/Makefile
+run-make/pdb-alt-path/Makefile
+run-make/pdb-buildinfo-cl-cmd/Makefile
+run-make/pgo-branch-weights/Makefile
+run-make/pgo-gen-lto/Makefile
+run-make/pgo-gen-no-imp-symbols/Makefile
+run-make/pgo-gen/Makefile
+run-make/pgo-indirect-call-promotion/Makefile
+run-make/pgo-use/Makefile
+run-make/pointer-auth-link-with-c/Makefile
+run-make/prefer-dylib/Makefile
+run-make/prefer-rlib/Makefile
+run-make/pretty-print-to-file/Makefile
+run-make/pretty-print-with-dep-file/Makefile
+run-make/print-calling-conventions/Makefile
+run-make/print-cfg/Makefile
+run-make/print-native-static-libs/Makefile
+run-make/print-target-list/Makefile
+run-make/profile/Makefile
+run-make/prune-link-args/Makefile
+run-make/raw-dylib-alt-calling-convention/Makefile
+run-make/raw-dylib-c/Makefile
+run-make/raw-dylib-cross-compilation/Makefile
+run-make/raw-dylib-custom-dlltool/Makefile
+run-make/raw-dylib-import-name-type/Makefile
+run-make/raw-dylib-inline-cross-dylib/Makefile
+run-make/raw-dylib-link-ordinal/Makefile
+run-make/raw-dylib-stdcall-ordinal/Makefile
+run-make/redundant-libs/Makefile
+run-make/relocation-model/Makefile
+run-make/relro-levels/Makefile
+run-make/remap-path-prefix-dwarf/Makefile
+run-make/remap-path-prefix/Makefile
+run-make/repr128-dwarf/Makefile
+run-make/reproducible-build-2/Makefile
+run-make/reproducible-build/Makefile
+run-make/resolve-rename/Makefile
+run-make/return-non-c-like-enum-from-c/Makefile
+run-make/return-non-c-like-enum/Makefile
+run-make/rlib-chain/Makefile
+run-make/rlib-format-packed-bundled-libs-2/Makefile
+run-make/rlib-format-packed-bundled-libs-3/Makefile
+run-make/rlib-format-packed-bundled-libs/Makefile
+run-make/rmeta-preferred/Makefile
+run-make/rust-lld-custom-target/Makefile
+run-make/rust-lld/Makefile
+run-make/rustc-macro-dep-files/Makefile
+run-make/rustdoc-determinism/Makefile
+run-make/rustdoc-error-lines/Makefile
+run-make/rustdoc-io-error/Makefile
+run-make/rustdoc-map-file/Makefile
+run-make/rustdoc-output-path/Makefile
+run-make/rustdoc-scrape-examples-invalid-expr/Makefile
+run-make/rustdoc-scrape-examples-macros/Makefile
+run-make/rustdoc-scrape-examples-multiple/Makefile
+run-make/rustdoc-scrape-examples-ordering/Makefile
+run-make/rustdoc-scrape-examples-remap/Makefile
+run-make/rustdoc-scrape-examples-test/Makefile
+run-make/rustdoc-scrape-examples-whitespace/Makefile
+run-make/rustdoc-shared-flags/Makefile
+run-make/rustdoc-target-spec-json-path/Makefile
+run-make/rustdoc-themes/Makefile
+run-make/rustdoc-verify-output-files/Makefile
+run-make/rustdoc-with-out-dir-option/Makefile
+run-make/rustdoc-with-output-option/Makefile
+run-make/rustdoc-with-short-out-dir-option/Makefile
+run-make/sanitizer-cdylib-link/Makefile
+run-make/sanitizer-dylib-link/Makefile
+run-make/sanitizer-staticlib-link/Makefile
+run-make/separate-link-fail/Makefile
+run-make/separate-link/Makefile
+run-make/sepcomp-cci-copies/Makefile
+run-make/sepcomp-inlining/Makefile
+run-make/sepcomp-separate/Makefile
+run-make/share-generics-dylib/Makefile
+run-make/short-ice/Makefile
+run-make/silly-file-names/Makefile
+run-make/simd-ffi/Makefile
+run-make/simple-dylib/Makefile
+run-make/simple-rlib/Makefile
+run-make/split-debuginfo/Makefile
+run-make/stable-symbol-names/Makefile
+run-make/static-dylib-by-default/Makefile
+run-make/static-extern-type/Makefile
+run-make/static-pie/Makefile
+run-make/static-unwinding/Makefile
+run-make/staticlib-blank-lib/Makefile
+run-make/staticlib-dylib-linkage/Makefile
+run-make/std-core-cycle/Makefile
+run-make/stdin-non-utf8/Makefile
+run-make/suspicious-library/Makefile
+run-make/symbol-mangling-hashed/Makefile
+run-make/symbol-visibility/Makefile
+run-make/symbols-include-type-name/Makefile
+run-make/symlinked-extern/Makefile
+run-make/symlinked-libraries/Makefile
+run-make/symlinked-rlib/Makefile
+run-make/sysroot-crates-are-unstable/Makefile
+run-make/target-cpu-native/Makefile
+run-make/target-specs/Makefile
+run-make/target-without-atomic-cas/Makefile
+run-make/test-benches/Makefile
+run-make/test-harness/Makefile
+run-make/thumb-none-cortex-m/Makefile
+run-make/thumb-none-qemu/Makefile
+run-make/track-path-dep-info/Makefile
+run-make/track-pgo-dep-info/Makefile
+run-make/translation/Makefile
+run-make/type-mismatch-same-crate-name/Makefile
+run-make/unknown-mod-stdin/Makefile
+run-make/unstable-flag-required/Makefile
+run-make/use-suggestions-rust-2018/Makefile
+run-make/used-cdylib-macos/Makefile
+run-make/used/Makefile
+run-make/valid-print-requests/Makefile
+run-make/version/Makefile
+run-make/volatile-intrinsics/Makefile
+run-make/wasm-exceptions-nostd/Makefile
+run-make/wasm-override-linker/Makefile
+run-make/weird-output-filenames/Makefile
+run-make/windows-binary-no-external-deps/Makefile
+run-make/windows-safeseh/Makefile
+run-make/windows-spawn/Makefile
+run-make/windows-subsystem/Makefile
+run-make/x86_64-fortanix-unknown-sgx-lvi/Makefile

src/tools/tidy/src/lib.rs

@@ -69,6 +69,7 @@ fn tidy_error(args: &str) -> std::io::Result<()> {
 pub(crate) mod iter_header;
 pub mod mir_opt_tests;
 pub mod pal;
+pub mod run_make_tests;
 pub mod rustdoc_css_themes;
 pub mod rustdoc_gui_tests;
 pub mod style;

src/tools/tidy/src/main.rs

@@ -103,6 +103,8 @@ macro_rules! check {
         check!(tests_revision_unpaired_stdout_stderr, &tests_path);
         check!(debug_artifacts, &tests_path);
         check!(ui_tests, &root_path, bless);
+        // FIXME(jieyouxu): remove this check once all run-make tests are ported over to rmake.rs.
+        check!(run_make_tests, &tests_path, &src_path, bless);
         check!(mir_opt_tests, &tests_path, bless);
         check!(rustdoc_gui_tests, &tests_path);
         check!(rustdoc_css_themes, &librustdoc_path);

src/tools/tidy/src/run_make_tests.rs

@@ -0,0 +1,89 @@
+//! Tidy check to ensure that no new Makefiles are added under `tests/run-make/`.
+
+use std::collections::BTreeSet;
+use std::fs::File;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+pub fn check(tests_path: &Path, src_path: &Path, bless: bool, bad: &mut bool) {
+    let allowed_makefiles = {
+        let allowed_makefiles = include_str!("allowed_run_make_makefiles.txt");
+        let allowed_makefiles = allowed_makefiles.lines().collect::<Vec<_>>();
+        let is_sorted = allowed_makefiles.windows(2).all(|w| w[0] < w[1]);
+        if !is_sorted && !bless {
+            tidy_error!(
+                bad,
+                "`src/tools/tidy/src/allowed_run_make_makefiles.txt` is not in order, likely \
+                because you modified it manually, please only update it with command \
+                `x test tidy --bless`"
+            );
+        }
+        let allowed_makefiles_unique =
+            allowed_makefiles.iter().map(ToString::to_string).collect::<BTreeSet<String>>();
+        if allowed_makefiles_unique.len() != allowed_makefiles.len() {
+            tidy_error!(
+                bad,
+                "`src/tools/tidy/src/allowed_run_make_makefiles.txt` contains duplicate entries, \
+                likely because you modified it manually, please only update it with command \
+                `x test tidy --bless`"
+            );
+        }
+        allowed_makefiles_unique
+    };
+
+    let mut remaining_makefiles = allowed_makefiles.clone();
+
+    crate::walk::walk_no_read(
+        &[tests_path.join("run-make").as_ref()],
+        |_, _| false,
+        &mut |entry| {
+            if entry.file_type().map_or(true, |t| t.is_dir()) {
+                return;
+            }
+
+            if entry.file_name().to_str().map_or(true, |f| f != "Makefile") {
+                return;
+            }
+
+            let makefile_path = entry.path().strip_prefix(&tests_path).unwrap();
+            let makefile_path = makefile_path.to_str().unwrap().replace('\\', "/");
+
+            if !remaining_makefiles.remove(&makefile_path) {
+                tidy_error!(
+                    bad,
+                    "found run-make Makefile not permitted in \
+                `src/tools/tidy/src/allowed_run_make_makefiles.txt`, please write new run-make \
+                tests with `rmake.rs` instead: {}",
+                    entry.path().display()
+                );
+            }
+        },
+    );
+
+    // If there are any expected Makefiles remaining, they were moved or deleted.
+    // Our data must remain up to date, so they must be removed from
+    // `src/tools/tidy/src/allowed_run_make_makefiles.txt`.
+    // This can be done automatically on --bless, or else a tidy error will be issued.
+    if bless && !remaining_makefiles.is_empty() {
+        let tidy_src = src_path.join("tools").join("tidy").join("src");
+        let org_file_path = tidy_src.join("allowed_run_make_makefiles.txt");
+        let temp_file_path = tidy_src.join("blessed_allowed_run_make_makefiles.txt");
+        let mut temp_file = t!(File::create_new(&temp_file_path));
+        for file in allowed_makefiles.difference(&remaining_makefiles) {
+            t!(writeln!(temp_file, "{file}"));
+        }
+        t!(std::fs::rename(&temp_file_path, &org_file_path));
+    } else {
+        for file in remaining_makefiles {
+            let mut p = PathBuf::from(tests_path);
+            p.push(file);
+            tidy_error!(
+                bad,
+                "Makefile `{}` no longer exists and should be removed from the exclusions in \
+                `src/tools/tidy/src/allowed_run_make_makefiles.txt`, you can run --bless to update \
+                the allow list",
+                p.display()
+            );
+        }
+    }
+}