diff --git a/Cargo.toml b/Cargo.toml
index 4993925489f..69deaca0b4b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,6 +32,7 @@ compiletest_rs = { version = "0.8", features = ["tmp"] }
 tester = "0.9"
 regex = "1.5"
 toml = "0.5"
+walkdir = "2.3"
 # This is used by the `collect-metadata` alias.
 filetime = "0.2"
 
diff --git a/tests/compile-test.rs b/tests/compile-test.rs
index 3615d07154d..92ac1a2be56 100644
--- a/tests/compile-test.rs
+++ b/tests/compile-test.rs
@@ -433,7 +433,7 @@ fn rustfix_coverage_known_exceptions_accuracy() {
         let rs_path = Path::new("tests/ui").join(filename);
         assert!(
             rs_path.exists(),
-            "`{}` does not exists",
+            "`{}` does not exist",
             rs_path.strip_prefix(env!("CARGO_MANIFEST_DIR")).unwrap().display()
         );
         let fixed_path = rs_path.with_extension("fixed");
@@ -445,6 +445,45 @@ fn rustfix_coverage_known_exceptions_accuracy() {
     }
 }
 
+#[test]
+fn ui_cargo_toml_metadata() {
+    let ui_cargo_path = Path::new("tests/ui-cargo");
+    let cargo_common_metadata_path = ui_cargo_path.join("cargo_common_metadata");
+    let publish_exceptions =
+        ["fail_publish", "fail_publish_true", "pass_publish_empty"].map(|path| cargo_common_metadata_path.join(path));
+
+    for entry in walkdir::WalkDir::new(ui_cargo_path) {
+        let entry = entry.unwrap();
+        let path = entry.path();
+        if path.file_name() != Some(OsStr::new("Cargo.toml")) {
+            continue;
+        }
+
+        let toml = fs::read_to_string(path).unwrap().parse::<toml::Value>().unwrap();
+
+        let package = toml.as_table().unwrap().get("package").unwrap().as_table().unwrap();
+
+        let name = package.get("name").unwrap().as_str().unwrap().replace('-', "_");
+        assert!(
+            path.parent()
+                .unwrap()
+                .components()
+                .map(|component| component.as_os_str().to_string_lossy().replace('-', "_"))
+                .any(|s| *s == name)
+                || path.starts_with(&cargo_common_metadata_path),
+            "{:?} has incorrect package name",
+            path
+        );
+
+        let publish = package.get("publish").and_then(toml::Value::as_bool).unwrap_or(true);
+        assert!(
+            !publish || publish_exceptions.contains(&path.parent().unwrap().to_path_buf()),
+            "{:?} lacks `publish = false`",
+            path
+        );
+    }
+}
+
 /// Restores an env var on drop
 #[must_use]
 struct VarGuard {
diff --git a/tests/ui-cargo/cargo_common_metadata/fail/Cargo.toml b/tests/ui-cargo/cargo_common_metadata/fail/Cargo.toml
index ae0a6032996..bc8e428f859 100644
--- a/tests/ui-cargo/cargo_common_metadata/fail/Cargo.toml
+++ b/tests/ui-cargo/cargo_common_metadata/fail/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "cargo_common_metadata_fail"
 version = "0.1.0"
 publish = false
 
diff --git a/tests/ui-cargo/cargo_common_metadata/fail/src/main.stderr b/tests/ui-cargo/cargo_common_metadata/fail/src/main.stderr
index 5e9aa8dc36a..86953142bef 100644
--- a/tests/ui-cargo/cargo_common_metadata/fail/src/main.stderr
+++ b/tests/ui-cargo/cargo_common_metadata/fail/src/main.stderr
@@ -1,16 +1,16 @@
-error: package `cargo_common_metadata` is missing `package.description` metadata
+error: package `cargo_common_metadata_fail` is missing `package.description` metadata
    |
    = note: `-D clippy::cargo-common-metadata` implied by `-D warnings`
 
-error: package `cargo_common_metadata` is missing `either package.license or package.license_file` metadata
+error: package `cargo_common_metadata_fail` is missing `either package.license or package.license_file` metadata
 
-error: package `cargo_common_metadata` is missing `package.repository` metadata
+error: package `cargo_common_metadata_fail` is missing `package.repository` metadata
 
-error: package `cargo_common_metadata` is missing `package.readme` metadata
+error: package `cargo_common_metadata_fail` is missing `package.readme` metadata
 
-error: package `cargo_common_metadata` is missing `package.keywords` metadata
+error: package `cargo_common_metadata_fail` is missing `package.keywords` metadata
 
-error: package `cargo_common_metadata` is missing `package.categories` metadata
+error: package `cargo_common_metadata_fail` is missing `package.categories` metadata
 
 error: aborting due to 6 previous errors
 
diff --git a/tests/ui-cargo/cargo_common_metadata/fail_publish/Cargo.toml b/tests/ui-cargo/cargo_common_metadata/fail_publish/Cargo.toml
index 7595696353c..5005b83f59d 100644
--- a/tests/ui-cargo/cargo_common_metadata/fail_publish/Cargo.toml
+++ b/tests/ui-cargo/cargo_common_metadata/fail_publish/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "cargo_common_metadata_fail_publish"
 version = "0.1.0"
 publish = ["some-registry-name"]
 
diff --git a/tests/ui-cargo/cargo_common_metadata/fail_publish/src/main.stderr b/tests/ui-cargo/cargo_common_metadata/fail_publish/src/main.stderr
index 5e9aa8dc36a..ac1b5e8e903 100644
--- a/tests/ui-cargo/cargo_common_metadata/fail_publish/src/main.stderr
+++ b/tests/ui-cargo/cargo_common_metadata/fail_publish/src/main.stderr
@@ -1,16 +1,16 @@
-error: package `cargo_common_metadata` is missing `package.description` metadata
+error: package `cargo_common_metadata_fail_publish` is missing `package.description` metadata
    |
    = note: `-D clippy::cargo-common-metadata` implied by `-D warnings`
 
-error: package `cargo_common_metadata` is missing `either package.license or package.license_file` metadata
+error: package `cargo_common_metadata_fail_publish` is missing `either package.license or package.license_file` metadata
 
-error: package `cargo_common_metadata` is missing `package.repository` metadata
+error: package `cargo_common_metadata_fail_publish` is missing `package.repository` metadata
 
-error: package `cargo_common_metadata` is missing `package.readme` metadata
+error: package `cargo_common_metadata_fail_publish` is missing `package.readme` metadata
 
-error: package `cargo_common_metadata` is missing `package.keywords` metadata
+error: package `cargo_common_metadata_fail_publish` is missing `package.keywords` metadata
 
-error: package `cargo_common_metadata` is missing `package.categories` metadata
+error: package `cargo_common_metadata_fail_publish` is missing `package.categories` metadata
 
 error: aborting due to 6 previous errors
 
diff --git a/tests/ui-cargo/cargo_common_metadata/fail_publish_true/Cargo.toml b/tests/ui-cargo/cargo_common_metadata/fail_publish_true/Cargo.toml
index 7e5b88383cc..51858eecd0a 100644
--- a/tests/ui-cargo/cargo_common_metadata/fail_publish_true/Cargo.toml
+++ b/tests/ui-cargo/cargo_common_metadata/fail_publish_true/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "cargo_common_metadata_fail_publish_true"
 version = "0.1.0"
 publish = true
 
diff --git a/tests/ui-cargo/cargo_common_metadata/fail_publish_true/src/main.stderr b/tests/ui-cargo/cargo_common_metadata/fail_publish_true/src/main.stderr
index 5e9aa8dc36a..be32c0dc418 100644
--- a/tests/ui-cargo/cargo_common_metadata/fail_publish_true/src/main.stderr
+++ b/tests/ui-cargo/cargo_common_metadata/fail_publish_true/src/main.stderr
@@ -1,16 +1,16 @@
-error: package `cargo_common_metadata` is missing `package.description` metadata
+error: package `cargo_common_metadata_fail_publish_true` is missing `package.description` metadata
    |
    = note: `-D clippy::cargo-common-metadata` implied by `-D warnings`
 
-error: package `cargo_common_metadata` is missing `either package.license or package.license_file` metadata
+error: package `cargo_common_metadata_fail_publish_true` is missing `either package.license or package.license_file` metadata
 
-error: package `cargo_common_metadata` is missing `package.repository` metadata
+error: package `cargo_common_metadata_fail_publish_true` is missing `package.repository` metadata
 
-error: package `cargo_common_metadata` is missing `package.readme` metadata
+error: package `cargo_common_metadata_fail_publish_true` is missing `package.readme` metadata
 
-error: package `cargo_common_metadata` is missing `package.keywords` metadata
+error: package `cargo_common_metadata_fail_publish_true` is missing `package.keywords` metadata
 
-error: package `cargo_common_metadata` is missing `package.categories` metadata
+error: package `cargo_common_metadata_fail_publish_true` is missing `package.categories` metadata
 
 error: aborting due to 6 previous errors
 
diff --git a/tests/ui-cargo/cargo_common_metadata/pass/Cargo.toml b/tests/ui-cargo/cargo_common_metadata/pass/Cargo.toml
index cb4774d43a2..9f6e51fb4d9 100644
--- a/tests/ui-cargo/cargo_common_metadata/pass/Cargo.toml
+++ b/tests/ui-cargo/cargo_common_metadata/pass/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "cargo_common_metadata_pass"
 version = "0.1.0"
 publish = false
 description = "A test package for the cargo_common_metadata lint"
diff --git a/tests/ui-cargo/cargo_common_metadata/pass_publish_empty/Cargo.toml b/tests/ui-cargo/cargo_common_metadata/pass_publish_empty/Cargo.toml
index 0a879c99b5b..828efee3a8f 100644
--- a/tests/ui-cargo/cargo_common_metadata/pass_publish_empty/Cargo.toml
+++ b/tests/ui-cargo/cargo_common_metadata/pass_publish_empty/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "cargo_common_metadata_pass_publish_empty"
 version = "0.1.0"
 publish = []
 
diff --git a/tests/ui-cargo/cargo_common_metadata/pass_publish_false/Cargo.toml b/tests/ui-cargo/cargo_common_metadata/pass_publish_false/Cargo.toml
index ae0a6032996..45a5bf7c574 100644
--- a/tests/ui-cargo/cargo_common_metadata/pass_publish_false/Cargo.toml
+++ b/tests/ui-cargo/cargo_common_metadata/pass_publish_false/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "cargo_common_metadata_pass_publish_false"
 version = "0.1.0"
 publish = false
 
diff --git a/tests/ui-cargo/cargo_rust_version/fail_both_diff/Cargo.toml b/tests/ui-cargo/cargo_rust_version/fail_both_diff/Cargo.toml
index 73ec29c5803..946d1b366f0 100644
--- a/tests/ui-cargo/cargo_rust_version/fail_both_diff/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/fail_both_diff/Cargo.toml
@@ -2,6 +2,7 @@
 name = "fail-both-diff"
 version = "0.1.0"
 rust-version = "1.56"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/fail_both_same/Cargo.toml b/tests/ui-cargo/cargo_rust_version/fail_both_same/Cargo.toml
index 2d6d547e4fe..46b92a1050e 100644
--- a/tests/ui-cargo/cargo_rust_version/fail_both_same/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/fail_both_same/Cargo.toml
@@ -2,6 +2,7 @@
 name = "fail-both-same"
 version = "0.1.0"
 rust-version = "1.57.0"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/fail_cargo/Cargo.toml b/tests/ui-cargo/cargo_rust_version/fail_cargo/Cargo.toml
index 36a53bd829d..189cc9f68dc 100644
--- a/tests/ui-cargo/cargo_rust_version/fail_cargo/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/fail_cargo/Cargo.toml
@@ -2,6 +2,7 @@
 name = "fail-cargo"
 version = "0.1.0"
 rust-version = "1.56.1"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/fail_clippy/Cargo.toml b/tests/ui-cargo/cargo_rust_version/fail_clippy/Cargo.toml
index 9f644a1a39a..bdb7f261d9e 100644
--- a/tests/ui-cargo/cargo_rust_version/fail_clippy/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/fail_clippy/Cargo.toml
@@ -1,6 +1,7 @@
 [package]
 name = "fail-clippy"
 version = "0.1.0"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/fail_file_attr/Cargo.toml b/tests/ui-cargo/cargo_rust_version/fail_file_attr/Cargo.toml
index 5380e993b29..84448ea41f6 100644
--- a/tests/ui-cargo/cargo_rust_version/fail_file_attr/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/fail_file_attr/Cargo.toml
@@ -2,6 +2,7 @@
 name = "fail-file-attr"
 version = "0.1.0"
 rust-version = "1.13"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/pass_both_same/Cargo.toml b/tests/ui-cargo/cargo_rust_version/pass_both_same/Cargo.toml
index 1f9bd8f9a84..809c0e74875 100644
--- a/tests/ui-cargo/cargo_rust_version/pass_both_same/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/pass_both_same/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "fail-both-same"
+name = "pass-both-same"
 version = "0.1.0"
 rust-version = "1.13.0"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/pass_cargo/Cargo.toml b/tests/ui-cargo/cargo_rust_version/pass_cargo/Cargo.toml
index 77538027c0f..32d404f842c 100644
--- a/tests/ui-cargo/cargo_rust_version/pass_cargo/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/pass_cargo/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "fail-cargo"
+name = "pass-cargo"
 version = "0.1.0"
 rust-version = "1.13.0"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/pass_clippy/Cargo.toml b/tests/ui-cargo/cargo_rust_version/pass_clippy/Cargo.toml
index 9f644a1a39a..cc937d6e625 100644
--- a/tests/ui-cargo/cargo_rust_version/pass_clippy/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/pass_clippy/Cargo.toml
@@ -1,6 +1,7 @@
 [package]
-name = "fail-clippy"
+name = "pass-clippy"
 version = "0.1.0"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/pass_file_attr/Cargo.toml b/tests/ui-cargo/cargo_rust_version/pass_file_attr/Cargo.toml
index f0387cd90b8..8ef689880d4 100644
--- a/tests/ui-cargo/cargo_rust_version/pass_file_attr/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/pass_file_attr/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "fail-file-attr"
+name = "pass-file-attr"
 version = "0.1.0"
 rust-version = "1.59"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/cargo_rust_version/warn_both_diff/Cargo.toml b/tests/ui-cargo/cargo_rust_version/warn_both_diff/Cargo.toml
index a19d5b33fe5..e9f94594f70 100644
--- a/tests/ui-cargo/cargo_rust_version/warn_both_diff/Cargo.toml
+++ b/tests/ui-cargo/cargo_rust_version/warn_both_diff/Cargo.toml
@@ -2,6 +2,7 @@
 name = "warn-both-diff"
 version = "0.1.0"
 rust-version = "1.56.0"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/module_style/fail_mod/Cargo.toml b/tests/ui-cargo/module_style/fail_mod/Cargo.toml
index 27b61c09fb4..b3d36a9fb64 100644
--- a/tests/ui-cargo/module_style/fail_mod/Cargo.toml
+++ b/tests/ui-cargo/module_style/fail_mod/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "fail"
+name = "fail-mod"
 version = "0.1.0"
 edition = "2018"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/module_style/fail_no_mod/Cargo.toml b/tests/ui-cargo/module_style/fail_no_mod/Cargo.toml
index 27b61c09fb4..3610d13c1f3 100644
--- a/tests/ui-cargo/module_style/fail_no_mod/Cargo.toml
+++ b/tests/ui-cargo/module_style/fail_no_mod/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "fail"
+name = "fail-no-mod"
 version = "0.1.0"
 edition = "2018"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/module_style/pass_mod/Cargo.toml b/tests/ui-cargo/module_style/pass_mod/Cargo.toml
index 27b61c09fb4..1c2991695bc 100644
--- a/tests/ui-cargo/module_style/pass_mod/Cargo.toml
+++ b/tests/ui-cargo/module_style/pass_mod/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "fail"
+name = "pass-mod"
 version = "0.1.0"
 edition = "2018"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/module_style/pass_no_mod/Cargo.toml b/tests/ui-cargo/module_style/pass_no_mod/Cargo.toml
index 3c0896dd2cd..4180aaf5185 100644
--- a/tests/ui-cargo/module_style/pass_no_mod/Cargo.toml
+++ b/tests/ui-cargo/module_style/pass_no_mod/Cargo.toml
@@ -1,7 +1,8 @@
 [package]
-name = "pass"
+name = "pass-no-mod"
 version = "0.1.0"
 edition = "2018"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/multiple_config_files/no_warn/Cargo.toml b/tests/ui-cargo/multiple_config_files/no_warn/Cargo.toml
index 79c973cbfd2..7eb56cc4e9d 100644
--- a/tests/ui-cargo/multiple_config_files/no_warn/Cargo.toml
+++ b/tests/ui-cargo/multiple_config_files/no_warn/Cargo.toml
@@ -2,6 +2,7 @@
 name = "no_warn"
 version = "0.1.0"
 edition = "2021"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/multiple_config_files/warn/Cargo.toml b/tests/ui-cargo/multiple_config_files/warn/Cargo.toml
index 3d5c707579b..b4847d070aa 100644
--- a/tests/ui-cargo/multiple_config_files/warn/Cargo.toml
+++ b/tests/ui-cargo/multiple_config_files/warn/Cargo.toml
@@ -2,6 +2,7 @@
 name = "warn"
 version = "0.1.0"
 edition = "2021"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/tests/ui-cargo/multiple_crate_versions/pass/Cargo.toml b/tests/ui-cargo/multiple_crate_versions/pass/Cargo.toml
index b4b49bb369a..6c46571c5bf 100644
--- a/tests/ui-cargo/multiple_crate_versions/pass/Cargo.toml
+++ b/tests/ui-cargo/multiple_crate_versions/pass/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "cargo_common_metadata"
+name = "multiple_crate_versions"
 version = "0.1.0"
 publish = false