From 2cb8c5fff690cff5e98c84f9bd08411f79347bd9 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Thu, 22 Feb 2018 18:59:04 -0600 Subject: [PATCH 01/17] Run rustfmt on tidy/src/deps.rs --- src/tools/tidy/src/deps.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 4dedf6bfe77..c3fb0d3913f 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -29,19 +29,19 @@ // tooling. It is _crucial_ that no exception crates be dependencies // of the Rust runtime (std / test). static EXCEPTIONS: &'static [&'static str] = &[ - "mdbook", // MPL2, mdbook - "openssl", // BSD+advertising clause, cargo, mdbook - "pest", // MPL2, mdbook via handlebars - "thread-id", // Apache-2.0, mdbook - "toml-query", // MPL-2.0, mdbook - "is-match", // MPL-2.0, mdbook - "cssparser", // MPL-2.0, rustdoc - "smallvec", // MPL-2.0, rustdoc + "mdbook", // MPL2, mdbook + "openssl", // BSD+advertising clause, cargo, mdbook + "pest", // MPL2, mdbook via handlebars + "thread-id", // Apache-2.0, mdbook + "toml-query", // MPL-2.0, mdbook + "is-match", // MPL-2.0, mdbook + "cssparser", // MPL-2.0, rustdoc + "smallvec", // MPL-2.0, rustdoc "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo - "fuchsia-zircon", // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir) - "cssparser-macros", // MPL-2.0, rustdoc - "selectors", // MPL-2.0, rustdoc - "clippy_lints", // MPL-2.0 rls + "fuchsia-zircon", // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir) + "cssparser-macros", // MPL-2.0, rustdoc + "selectors", // MPL-2.0, rustdoc + "clippy_lints", // MPL-2.0 rls ]; pub fn check(path: &Path, bad: &mut bool) { @@ -57,7 +57,8 @@ pub fn check(path: &Path, bad: &mut bool) { if dir.path() .to_str() .unwrap() - .contains(&format!("src/vendor/{}", exception)) { + .contains(&format!("src/vendor/{}", exception)) + { continue 'next_path; } } @@ -102,7 +103,7 @@ fn extract_license(line: &str) -> String { let first_quote = line.find('"'); let last_quote = line.rfind('"'); if let (Some(f), Some(l)) = (first_quote, last_quote) { - let license = &line[f + 1 .. l]; + let license = &line[f + 1..l]; license.into() } else { "bad-license-parse".into() From 3ee410498db9826c3ccb81ce8c0aa40f7f3b4082 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Thu, 22 Feb 2018 19:52:56 -0600 Subject: [PATCH 02/17] Start adding a whitelist for rustc dependencies --- src/Cargo.lock | 5 +++ src/tools/tidy/Cargo.toml | 5 +++ src/tools/tidy/src/deps.rs | 76 +++++++++++++++++++++++++++++++++----- src/tools/tidy/src/lib.rs | 5 +++ 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 7620fe8ddb3..5e7909ff435 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -2585,6 +2585,11 @@ dependencies = [ [[package]] name = "tidy" version = "0.1.0" +dependencies = [ + "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "time" diff --git a/src/tools/tidy/Cargo.toml b/src/tools/tidy/Cargo.toml index 664aecfcbdb..f7b491823f8 100644 --- a/src/tools/tidy/Cargo.toml +++ b/src/tools/tidy/Cargo.toml @@ -2,3 +2,8 @@ name = "tidy" version = "0.1.0" authors = ["Alex Crichton "] + +[dependencies] +serde = "1.0.8" +serde_derive = "1.0.8" +serde_json = "1.0.2" diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index c3fb0d3913f..2cb8a70844e 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -14,6 +14,10 @@ use std::io::Read; use std::path::Path; +use std::process::Command; + +use serde_json; + static LICENSES: &'static [&'static str] = &[ "MIT/Apache-2.0", "MIT / Apache-2.0", @@ -44,31 +48,68 @@ "clippy_lints", // MPL-2.0 rls ]; +// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. +static WHITELIST: &'static [(&'static str, &'static str)] = &[]; + +// Some type for Serde to deserialize the output of `cargo metadata` to... + +#[derive(Deserialize)] +struct Output { + packages: Vec, + _resolve: String, +} + +#[derive(Deserialize)] +struct Package { + _id: String, + name: String, + version: String, + _source: Option, + _manifest_path: String, +} + +/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. +/// +/// Specifically, this checks that the license is correct and that the dependencies are on the +/// whitelist. pub fn check(path: &Path, bad: &mut bool) { + // Check licences let path = path.join("vendor"); assert!(path.exists(), "vendor directory missing"); let mut saw_dir = false; - 'next_path: for dir in t!(path.read_dir()) { + for dir in t!(path.read_dir()) { saw_dir = true; let dir = t!(dir); // skip our exceptions - for exception in EXCEPTIONS { - if dir.path() + if EXCEPTIONS.iter().any(|exception| { + dir.path() .to_str() .unwrap() .contains(&format!("src/vendor/{}", exception)) - { - continue 'next_path; - } + }) { + continue; } let toml = dir.path().join("Cargo.toml"); - if !check_license(&toml) { - *bad = true; - } + *bad = *bad || !check_license(&toml); } assert!(saw_dir, "no vendored source"); + + // Check dependencies + let deps = get_deps(&path); + *bad = *bad + || deps.iter().any( + |&Package { + ref name, + ref version, + .. + }| { + WHITELIST + .iter() + .all(|&(wname, wversion)| name != wname || version != wversion) + }, + ); } fn check_license(path: &Path) -> bool { @@ -109,3 +150,20 @@ fn extract_license(line: &str) -> String { "bad-license-parse".into() } } + +fn get_deps(path: &Path) -> Vec { + // Run `cargo metadata` to get the set of dependencies + let output = Command::new("cargo") + .arg("metadata") + .arg("--format-version") + .arg("1") + .arg("--manifest-path") + .arg(path.join("Cargo.toml")) + .output() + .expect("Unable to run `cargo metadata`") + .stdout; + let output = String::from_utf8_lossy(&output); + let output: Output = serde_json::from_str(&output).unwrap(); + + output.packages +} diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs index 5134c869912..c927ff19b27 100644 --- a/src/tools/tidy/src/lib.rs +++ b/src/tools/tidy/src/lib.rs @@ -15,6 +15,11 @@ #![deny(warnings)] +extern crate serde; +extern crate serde_json; +#[macro_use] +extern crate serde_derive; + use std::fs; use std::path::Path; From d62621839a64352e41f8908021cfe8e7d91f064a Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Thu, 22 Feb 2018 19:57:55 -0600 Subject: [PATCH 03/17] Comments --- src/tools/tidy/src/deps.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 2cb8a70844e..30eadd9ec9d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -51,7 +51,7 @@ // Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [(&'static str, &'static str)] = &[]; -// Some type for Serde to deserialize the output of `cargo metadata` to... +// Some types for Serde to deserialize the output of `cargo metadata` to... #[derive(Deserialize)] struct Output { @@ -151,6 +151,7 @@ fn extract_license(line: &str) -> String { } } +/// Get the dependencies of the crate at the given path using `cargo metadata`. fn get_deps(path: &Path) -> Vec { // Run `cargo metadata` to get the set of dependencies let output = Command::new("cargo") From 3570b9df6ae615691bfe296b5af816332a25299b Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Fri, 23 Feb 2018 18:01:51 -0600 Subject: [PATCH 04/17] MAKE IT FAILgit statusgit status --- src/tools/tidy/src/deps.rs | 72 +++++++++++++++++++++++++++----------- src/tools/tidy/src/main.rs | 1 + 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 30eadd9ec9d..a2d42a00c36 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -10,10 +10,10 @@ //! Check license of third-party deps by inspecting src/vendor +use std::collections::HashSet; use std::fs::File; use std::io::Read; use std::path::Path; - use std::process::Command; use serde_json; @@ -56,22 +56,40 @@ #[derive(Deserialize)] struct Output { packages: Vec, - _resolve: String, + + // Not used, but needed to not confuse serde :P + #[allow(dead_code)] resolve: Resolve, } #[derive(Deserialize)] struct Package { - _id: String, name: String, version: String, - _source: Option, - _manifest_path: String, + + // Not used, but needed to not confuse serde :P + #[allow(dead_code)] id: String, + #[allow(dead_code)] source: Option, + #[allow(dead_code)] manifest_path: String, +} + +// Not used, but needed to not confuse serde :P +#[allow(dead_code)] +#[derive(Deserialize)] +struct Resolve { + nodes: Vec, +} + +// Not used, but needed to not confuse serde :P +#[allow(dead_code)] +#[derive(Deserialize)] +struct ResolveNode { + id: String, + dependencies: Vec, } /// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. /// -/// Specifically, this checks that the license is correct and that the dependencies are on the -/// whitelist. +/// Specifically, this checks that the license is correct. pub fn check(path: &Path, bad: &mut bool) { // Check licences let path = path.join("vendor"); @@ -95,21 +113,35 @@ pub fn check(path: &Path, bad: &mut bool) { *bad = *bad || !check_license(&toml); } assert!(saw_dir, "no vendored source"); +} +/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. +/// +/// Specifically, this checks that the dependencies are on the whitelist. +pub fn check_whitelist(path: &Path, bad: &mut bool) { // Check dependencies - let deps = get_deps(&path); - *bad = *bad - || deps.iter().any( - |&Package { - ref name, - ref version, - .. - }| { - WHITELIST - .iter() - .all(|&(wname, wversion)| name != wname || version != wversion) - }, - ); + let deps: HashSet<_> = get_deps(&path) + .into_iter() + .map(|Package { name, version, .. }| (name, version)) + .collect(); + let whitelist: HashSet<(String, String)> = WHITELIST + .iter() + .map(|&(n, v)| (n.to_owned(), v.to_owned())) + .collect(); + + // Dependencies not in the whitelist + let mut unapproved: Vec<_> = deps.difference(&whitelist).collect(); + + // For ease of reading + unapproved.sort(); + + if unapproved.len() > 0 { + println!("Dependencies not on the whitelist:"); + for dep in unapproved { + println!("* {} {}", dep.0, dep.1); // name version + } + *bad = true; + } } fn check_license(path: &Path) -> bool { diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index f6640c902bc..f64536e2f59 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -41,6 +41,7 @@ fn main() { if !args.iter().any(|s| *s == "--no-vendor") { deps::check(&path, &mut bad); } + deps::check_whitelist(&path, &mut bad); if bad { eprintln!("some tidy checks failed"); From 6e016c587ff4feae9f2401c744f6d69fa3e22c77 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Fri, 23 Feb 2018 18:25:21 -0600 Subject: [PATCH 05/17] Trying to get paths right... --- src/tools/tidy/src/deps.rs | 311 ++++++++++++++++++++++++++++++++++++- 1 file changed, 310 insertions(+), 1 deletion(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index a2d42a00c36..b7f697f39ae 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -49,7 +49,315 @@ ]; // Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. -static WHITELIST: &'static [(&'static str, &'static str)] = &[]; +static WHITELIST: &'static [(&'static str, &'static str)] = &[ +// ("advapi32-sys", "0.2.0"), +// ("aho-corasick", "0.5.3"), +// ("aho-corasick", "0.6.4"), +// ("alloc", "0.0.0"), +// ("alloc_jemalloc", "0.0.0"), +// ("alloc_system", "0.0.0"), +// ("ansi_term", "0.10.2"), +// ("ar", "0.3.1"), +// ("arena", "0.0.0"), +// ("atty", "0.2.6"), +// ("backtrace", "0.3.5"), +// ("backtrace-sys", "0.1.16"), +// ("bin_lib", "0.1.0"), +// ("bitflags", "0.7.0"), +// ("bitflags", "0.9.1"), +// ("bitflags", "1.0.1"), +// ("bootstrap", "0.0.0"), +// ("borrow_error", "0.1.0"), +// ("bufstream", "0.1.3"), +// ("build-manifest", "0.1.0"), +// ("build_helper", "0.1.0"), +// ("byteorder", "1.2.1"), +// ("cargo", "0.26.0"), +// ("cargo_metadata", "0.2.3"), +// ("cargo_metadata", "0.4.0"), +// ("cargotest", "0.1.0"), +// ("cargotest2", "0.1.0"), +// ("cc", "1.0.4"), +// ("cfg-if", "0.1.2"), +// ("chrono", "0.4.0"), +// ("clap", "2.29.0"), +// ("clippy", "0.0.186"), +// ("clippy-mini-macro-test", "0.2.0"), +// ("clippy_lints", "0.0.186"), +// ("cmake", "0.1.29"), +// ("coco", "0.1.1"), +// ("commoncrypto", "0.2.0"), +// ("commoncrypto-sys", "0.2.0"), +// ("compiler_builtins", "0.0.0"), +// ("compiletest", "0.0.0"), +// ("compiletest_rs", "0.3.6"), +// ("completion", "0.1.0"), +// ("core", "0.0.0"), +// ("core-foundation", "0.4.6"), +// ("core-foundation-sys", "0.4.6"), +// ("crates-io", "0.15.0"), +// ("crossbeam", "0.2.12"), +// ("crossbeam", "0.3.2"), +// ("crypto-hash", "0.3.0"), +// ("curl", "0.4.11"), +// ("curl-sys", "0.4.1"), +// ("deglob", "0.1.0"), +// ("derive-new", "0.5.0"), +// ("diff", "0.1.11"), +// ("dlmalloc", "0.0.0"), +// ("docopt", "0.8.3"), +// ("dtoa", "0.4.2"), +// ("duct", "0.8.2"), +// ("either", "1.4.0"), +// ("endian-type", "0.1.2"), +// ("enum_primitive", "0.1.1"), +// ("env_logger", "0.3.5"), +// ("env_logger", "0.4.3"), +// ("env_logger", "0.5.3"), +// ("error-chain", "0.11.0"), +// ("error-chain", "0.8.1"), +// ("error_index_generator", "0.0.0"), +// ("failure", "0.1.1"), +// ("failure_derive", "0.1.1"), +// ("features", "0.1.0"), +// ("filetime", "0.1.15"), +// ("find_all_refs_no_cfg_test", "0.1.0"), +// ("find_impls", "0.1.0"), +// ("flate2", "1.0.1"), +// ("fmt_macros", "0.0.0"), +// ("fnv", "1.0.6"), +// ("foreign-types", "0.3.2"), +// ("foreign-types-shared", "0.1.1"), +// ("fs2", "0.4.3"), +// ("fuchsia-zircon", "0.3.3"), +// ("fuchsia-zircon-sys", "0.3.3"), +// ("futures", "0.1.17"), +// ("getopts", "0.2.15"), +// ("git2", "0.6.11"), +// ("git2-curl", "0.7.0"), +// ("glob", "0.2.11"), +// ("globset", "0.2.1"), +// ("graphviz", "0.0.0"), +// ("hamcrest", "0.1.1"), +// ("handlebars", "0.29.1"), +// ("hex", "0.2.0"), +// ("hex", "0.3.1"), +// ("home", "0.3.0"), +// ("idna", "0.1.4"), +// ("if_chain", "0.1.2"), +// ("ignore", "0.3.1"), +// ("infer_bin", "0.1.0"), +// ("infer_custom_bin", "0.1.0"), +// ("infer_lib", "0.1.0"), +// ("installer", "0.0.0"), +// ("is-match", "0.1.0"), +// ("itertools", "0.6.5"), +// ("itertools", "0.7.6"), +// ("itoa", "0.3.4"), +// ("jobserver", "0.1.9"), +// ("json", "0.11.12"), +// ("jsonrpc-core", "8.0.1"), +// ("kernel32-sys", "0.2.2"), +// ("languageserver-types", "0.30.0"), +// ("lazy_static", "0.2.11"), +// ("lazy_static", "1.0.0"), +// ("lazycell", "0.5.1"), +// ("libc", "0.0.0"), +// ("libc", "0.2.36"), +// ("libgit2-sys", "0.6.19"), +// ("libssh2-sys", "0.2.6"), +// ("libz-sys", "1.0.18"), +// ("linkchecker", "0.1.0"), +// ("log", "0.3.9"), +// ("log", "0.4.1"), +// ("log_settings", "0.1.1"), +// ("lzma-sys", "0.1.9"), +// ("matches", "0.1.6"), +// ("mdbook", "0.1.2"), +// ("memchr", "0.1.11"), +// ("memchr", "2.0.1"), +// ("miniz-sys", "0.1.10"), +// ("miow", "0.2.1"), +// ("miri", "0.1.0"), +// ("multiple_bins", "0.1.0"), +// ("net2", "0.2.31"), +// ("nibble_vec", "0.0.3"), +// ("nix", "0.8.1"), +// ("num", "0.1.41"), +// ("num-bigint", "0.1.41"), +// ("num-complex", "0.1.41"), +// ("num-integer", "0.1.35"), +// ("num-iter", "0.1.34"), +// ("num-rational", "0.1.40"), +// ("num-traits", "0.1.41"), +// ("num_cpus", "1.8.0"), +// ("open", "1.2.1"), +// ("openssl", "0.9.23"), +// ("openssl-probe", "0.1.2"), +// ("openssl-sys", "0.9.24"), +// ("os_pipe", "0.5.1"), +// ("owning_ref", "0.3.3"), +// ("panic_abort", "0.0.0"), +// ("panic_unwind", "0.0.0"), +// ("parking_lot", "0.5.3"), +// ("parking_lot_core", "0.2.9"), +// ("percent-encoding", "1.0.1"), +// ("pest", "0.3.3"), +// ("pkg-config", "0.3.9"), +// ("proc_macro", "0.0.0"), +// ("profiler_builtins", "0.0.0"), +// ("pulldown-cmark", "0.0.15"), +// ("pulldown-cmark", "0.1.0"), +// ("quick-error", "1.2.1"), +// ("quine-mc_cluskey", "0.2.4"), +// ("quote", "0.3.15"), +// ("racer", "2.0.12"), +// ("radix_trie", "0.1.2"), +// ("rand", "0.3.20"), +// ("rayon", "0.9.0"), +// ("rayon-core", "1.3.0"), +// ("redox_syscall", "0.1.37"), +// ("redox_termios", "0.1.1"), +// ("reformat", "0.1.0"), +// ("reformat_with_range", "0.1.0"), +// ("regex", "0.1.80"), +// ("regex", "0.2.5"), +// ("regex-syntax", "0.3.9"), +// ("regex-syntax", "0.4.2"), +// ("remote-test-client", "0.1.0"), +// ("remote-test-server", "0.1.0"), +// ("rls", "0.125.0"), +// ("rls-analysis", "0.11.0"), +// ("rls-blacklist", "0.1.0"), +// ("rls-data", "0.15.0"), +// ("rls-rustc", "0.2.1"), +// ("rls-span", "0.4.0"), +// ("rls-vfs", "0.4.4"), +// ("rustbook", "0.1.0"), +// ("rustc", "0.0.0"), +// ("rustc-ap-rustc_cratesio_shim", "29.0.0"), +// ("rustc-ap-rustc_data_structures", "29.0.0"), +// ("rustc-ap-rustc_errors", "29.0.0"), +// ("rustc-ap-serialize", "29.0.0"), +// ("rustc-ap-syntax", "29.0.0"), +// ("rustc-ap-syntax_pos", "29.0.0"), +// ("rustc-demangle", "0.1.5"), +// ("rustc-main", "0.0.0"), +// ("rustc-serialize", "0.3.24"), +// ("rustc_allocator", "0.0.0"), +// ("rustc_apfloat", "0.0.0"), +// ("rustc_asan", "0.0.0"), +// ("rustc_back", "0.0.0"), +// ("rustc_binaryen", "0.0.0"), +// ("rustc_borrowck", "0.0.0"), +// ("rustc_const_eval", "0.0.0"), +// ("rustc_const_math", "0.0.0"), +// ("rustc_cratesio_shim", "0.0.0"), +// ("rustc_data_structures", "0.0.0"), +// ("rustc_driver", "0.0.0"), +// ("rustc_errors", "0.0.0"), +// ("rustc_incremental", "0.0.0"), +// ("rustc_lint", "0.0.0"), +// ("rustc_llvm", "0.0.0"), +// ("rustc_lsan", "0.0.0"), +// ("rustc_metadata", "0.0.0"), +// ("rustc_mir", "0.0.0"), +// ("rustc_msan", "0.0.0"), +// ("rustc_passes", "0.0.0"), +// ("rustc_platform_intrinsics", "0.0.0"), +// ("rustc_plugin", "0.0.0"), +// ("rustc_privacy", "0.0.0"), +// ("rustc_resolve", "0.0.0"), +// ("rustc_save_analysis", "0.0.0"), +// ("rustc_trans", "0.0.0"), +// ("rustc_trans_utils", "0.0.0"), +// ("rustc_tsan", "0.0.0"), +// ("rustc_typeck", "0.0.0"), +// ("rustdoc", "0.0.0"), +// ("rustdoc-themes", "0.1.0"), +// ("rustdoc-tool", "0.0.0"), +// ("rustfmt-nightly", "0.3.8"), +// ("same-file", "0.1.3"), +// ("same-file", "1.0.2"), +// ("schannel", "0.1.10"), +// ("scoped-tls", "0.1.0"), +// ("scopeguard", "0.1.2"), +// ("scopeguard", "0.3.3"), +// ("semver", "0.6.0"), +// ("semver", "0.8.0"), +// ("semver", "0.9.0"), +// ("semver-parser", "0.7.0"), +// ("serde", "1.0.27"), +// ("serde_derive", "1.0.27"), +// ("serde_derive_internals", "0.19.0"), +// ("serde_ignored", "0.0.4"), +// ("serde_json", "1.0.9"), +// ("serialize", "0.0.0"), +// ("shared_child", "0.2.1"), +// ("shell-escape", "0.1.3"), +// ("shlex", "0.1.1"), +// ("smallvec", "0.6.0"), +// ("socket2", "0.3.0"), +// ("stable_deref_trait", "1.0.0"), +// ("std", "0.0.0"), +// ("std_unicode", "0.0.0"), +// ("strsim", "0.6.0"), +// ("syn", "0.11.11"), +// ("synom", "0.11.3"), +// ("synstructure", "0.6.1"), +// ("syntax", "0.0.0"), +// ("syntax_ext", "0.0.0"), +// ("syntax_pos", "0.0.0"), +// ("syntex_errors", "0.52.0"), +// ("syntex_pos", "0.52.0"), +// ("syntex_syntax", "0.52.0"), +// ("tar", "0.4.14"), +// ("tempdir", "0.3.5"), +// ("term", "0.0.0"), +// ("term", "0.4.6"), +// ("termcolor", "0.3.3"), +// ("termion", "1.5.1"), +// ("test", "0.0.0"), +// ("textwrap", "0.9.0"), +// ("thread-id", "2.0.0"), +// ("thread_local", "0.2.7"), +// ("thread_local", "0.3.5"), +// ("tidy", "0.1.0"), +// ("time", "0.1.39"), +// ("toml", "0.2.1"), +// ("toml", "0.4.5"), +// ("toml-query", "0.6.0"), +// ("unicode-bidi", "0.3.4"), +// ("unicode-normalization", "0.1.5"), +// ("unicode-segmentation", "1.2.0"), +// ("unicode-width", "0.1.4"), +// ("unicode-xid", "0.0.3"), +// ("unicode-xid", "0.0.4"), +// ("unreachable", "1.0.0"), +// ("unstable-book-gen", "0.1.0"), +// ("unwind", "0.0.0"), +// ("url", "1.6.0"), +// ("url_serde", "0.2.0"), +// ("userenv-sys", "0.2.0"), +// ("utf8-ranges", "0.1.3"), +// ("utf8-ranges", "1.0.0"), +// ("vcpkg", "0.2.2"), +// ("vec_map", "0.8.0"), +// ("void", "1.0.2"), +// ("walkdir", "1.0.7"), +// ("walkdir", "2.0.1"), +// ("winapi", "0.2.8"), +// ("winapi", "0.3.4"), +// ("winapi-build", "0.1.1"), +// ("winapi-i686-pc-windows-gnu", "0.4.0"), +// ("winapi-x86_64-pc-windows-gnu", "0.4.0"), +// ("wincolor", "0.1.4"), +// ("workspace_symbol", "0.1.0"), +// ("ws2_32-sys", "0.2.1"), +// ("xattr", "0.1.11"), +// ("xz2", "0.1.4"), +// ("yaml-rust", "0.3.5"), +]; // Some types for Serde to deserialize the output of `cargo metadata` to... @@ -186,6 +494,7 @@ fn extract_license(line: &str) -> String { /// Get the dependencies of the crate at the given path using `cargo metadata`. fn get_deps(path: &Path) -> Vec { // Run `cargo metadata` to get the set of dependencies + println!("Getting metadata from {:?}", path.join("Cargo.toml")); let output = Command::new("cargo") .arg("metadata") .arg("--format-version") From b9b1c378c5399d15eeab6c0347f97d9b32269270 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Mon, 26 Feb 2018 11:05:43 -0600 Subject: [PATCH 06/17] Get the path to cargo from rustbuild --- src/bootstrap/test.rs | 1 + src/tools/tidy/src/deps.rs | 9 ++++----- src/tools/tidy/src/main.rs | 7 +++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index c0998c1e42c..48490493525 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -528,6 +528,7 @@ fn run(self, builder: &Builder) { println!("tidy check ({})", host); let mut cmd = builder.tool_cmd(Tool::Tidy); cmd.arg(build.src.join("src")); + cmd.arg(&build.initial_cargo); if !build.config.vendor { cmd.arg("--no-vendor"); } diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index b7f697f39ae..4c9c0d8cf66 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -426,9 +426,9 @@ pub fn check(path: &Path, bad: &mut bool) { /// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. /// /// Specifically, this checks that the dependencies are on the whitelist. -pub fn check_whitelist(path: &Path, bad: &mut bool) { +pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { // Check dependencies - let deps: HashSet<_> = get_deps(&path) + let deps: HashSet<_> = get_deps(&path, &cargo) .into_iter() .map(|Package { name, version, .. }| (name, version)) .collect(); @@ -492,10 +492,9 @@ fn extract_license(line: &str) -> String { } /// Get the dependencies of the crate at the given path using `cargo metadata`. -fn get_deps(path: &Path) -> Vec { +fn get_deps(path: &Path, cargo: &Path) -> Vec { // Run `cargo metadata` to get the set of dependencies - println!("Getting metadata from {:?}", path.join("Cargo.toml")); - let output = Command::new("cargo") + let output = Command::new(cargo) .arg("metadata") .arg("--format-version") .arg("1") diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index f64536e2f59..afa3ebd1983 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -24,9 +24,12 @@ use std::env; fn main() { - let path = env::args_os().skip(1).next().expect("need an argument"); + let path = env::args_os().skip(1).next().expect("need path to src"); let path = PathBuf::from(path); + let cargo = env::args_os().skip(2).next().expect("need path to cargo"); + let cargo = PathBuf::from(cargo); + let args: Vec = env::args().skip(1).collect(); let mut bad = false; @@ -41,7 +44,7 @@ fn main() { if !args.iter().any(|s| *s == "--no-vendor") { deps::check(&path, &mut bad); } - deps::check_whitelist(&path, &mut bad); + deps::check_whitelist(&path, &cargo, &mut bad); if bad { eprintln!("some tidy checks failed"); From 50876d1ca47ab34473b70c7364b1bfb835f5c25a Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Mon, 26 Feb 2018 23:45:04 -0600 Subject: [PATCH 07/17] Only check the whitelist for some crates --- src/tools/tidy/src/deps.rs | 495 +++++++++++-------------------------- 1 file changed, 151 insertions(+), 344 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 4c9c0d8cf66..0d351c9e735 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -28,10 +28,10 @@ "Unlicense/MIT", ]; -// These are exceptions to Rust's permissive licensing policy, and -// should be considered bugs. Exceptions are only allowed in Rust -// tooling. It is _crucial_ that no exception crates be dependencies -// of the Rust runtime (std / test). +/// These are exceptions to Rust's permissive licensing policy, and +/// should be considered bugs. Exceptions are only allowed in Rust +/// tooling. It is _crucial_ that no exception crates be dependencies +/// of the Rust runtime (std / test). static EXCEPTIONS: &'static [&'static str] = &[ "mdbook", // MPL2, mdbook "openssl", // BSD+advertising clause, cargo, mdbook @@ -48,353 +48,122 @@ "clippy_lints", // MPL-2.0 rls ]; -// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. -static WHITELIST: &'static [(&'static str, &'static str)] = &[ -// ("advapi32-sys", "0.2.0"), -// ("aho-corasick", "0.5.3"), -// ("aho-corasick", "0.6.4"), -// ("alloc", "0.0.0"), -// ("alloc_jemalloc", "0.0.0"), -// ("alloc_system", "0.0.0"), -// ("ansi_term", "0.10.2"), -// ("ar", "0.3.1"), -// ("arena", "0.0.0"), -// ("atty", "0.2.6"), -// ("backtrace", "0.3.5"), -// ("backtrace-sys", "0.1.16"), -// ("bin_lib", "0.1.0"), -// ("bitflags", "0.7.0"), -// ("bitflags", "0.9.1"), -// ("bitflags", "1.0.1"), -// ("bootstrap", "0.0.0"), -// ("borrow_error", "0.1.0"), -// ("bufstream", "0.1.3"), -// ("build-manifest", "0.1.0"), -// ("build_helper", "0.1.0"), -// ("byteorder", "1.2.1"), -// ("cargo", "0.26.0"), -// ("cargo_metadata", "0.2.3"), -// ("cargo_metadata", "0.4.0"), -// ("cargotest", "0.1.0"), -// ("cargotest2", "0.1.0"), -// ("cc", "1.0.4"), -// ("cfg-if", "0.1.2"), -// ("chrono", "0.4.0"), -// ("clap", "2.29.0"), -// ("clippy", "0.0.186"), -// ("clippy-mini-macro-test", "0.2.0"), -// ("clippy_lints", "0.0.186"), -// ("cmake", "0.1.29"), -// ("coco", "0.1.1"), -// ("commoncrypto", "0.2.0"), -// ("commoncrypto-sys", "0.2.0"), -// ("compiler_builtins", "0.0.0"), -// ("compiletest", "0.0.0"), -// ("compiletest_rs", "0.3.6"), -// ("completion", "0.1.0"), -// ("core", "0.0.0"), -// ("core-foundation", "0.4.6"), -// ("core-foundation-sys", "0.4.6"), -// ("crates-io", "0.15.0"), -// ("crossbeam", "0.2.12"), -// ("crossbeam", "0.3.2"), -// ("crypto-hash", "0.3.0"), -// ("curl", "0.4.11"), -// ("curl-sys", "0.4.1"), -// ("deglob", "0.1.0"), -// ("derive-new", "0.5.0"), -// ("diff", "0.1.11"), -// ("dlmalloc", "0.0.0"), -// ("docopt", "0.8.3"), -// ("dtoa", "0.4.2"), -// ("duct", "0.8.2"), -// ("either", "1.4.0"), -// ("endian-type", "0.1.2"), -// ("enum_primitive", "0.1.1"), -// ("env_logger", "0.3.5"), -// ("env_logger", "0.4.3"), -// ("env_logger", "0.5.3"), -// ("error-chain", "0.11.0"), -// ("error-chain", "0.8.1"), -// ("error_index_generator", "0.0.0"), -// ("failure", "0.1.1"), -// ("failure_derive", "0.1.1"), -// ("features", "0.1.0"), -// ("filetime", "0.1.15"), -// ("find_all_refs_no_cfg_test", "0.1.0"), -// ("find_impls", "0.1.0"), -// ("flate2", "1.0.1"), -// ("fmt_macros", "0.0.0"), -// ("fnv", "1.0.6"), -// ("foreign-types", "0.3.2"), -// ("foreign-types-shared", "0.1.1"), -// ("fs2", "0.4.3"), -// ("fuchsia-zircon", "0.3.3"), -// ("fuchsia-zircon-sys", "0.3.3"), -// ("futures", "0.1.17"), -// ("getopts", "0.2.15"), -// ("git2", "0.6.11"), -// ("git2-curl", "0.7.0"), -// ("glob", "0.2.11"), -// ("globset", "0.2.1"), -// ("graphviz", "0.0.0"), -// ("hamcrest", "0.1.1"), -// ("handlebars", "0.29.1"), -// ("hex", "0.2.0"), -// ("hex", "0.3.1"), -// ("home", "0.3.0"), -// ("idna", "0.1.4"), -// ("if_chain", "0.1.2"), -// ("ignore", "0.3.1"), -// ("infer_bin", "0.1.0"), -// ("infer_custom_bin", "0.1.0"), -// ("infer_lib", "0.1.0"), -// ("installer", "0.0.0"), -// ("is-match", "0.1.0"), -// ("itertools", "0.6.5"), -// ("itertools", "0.7.6"), -// ("itoa", "0.3.4"), -// ("jobserver", "0.1.9"), -// ("json", "0.11.12"), -// ("jsonrpc-core", "8.0.1"), -// ("kernel32-sys", "0.2.2"), -// ("languageserver-types", "0.30.0"), -// ("lazy_static", "0.2.11"), -// ("lazy_static", "1.0.0"), -// ("lazycell", "0.5.1"), -// ("libc", "0.0.0"), -// ("libc", "0.2.36"), -// ("libgit2-sys", "0.6.19"), -// ("libssh2-sys", "0.2.6"), -// ("libz-sys", "1.0.18"), -// ("linkchecker", "0.1.0"), -// ("log", "0.3.9"), -// ("log", "0.4.1"), -// ("log_settings", "0.1.1"), -// ("lzma-sys", "0.1.9"), -// ("matches", "0.1.6"), -// ("mdbook", "0.1.2"), -// ("memchr", "0.1.11"), -// ("memchr", "2.0.1"), -// ("miniz-sys", "0.1.10"), -// ("miow", "0.2.1"), -// ("miri", "0.1.0"), -// ("multiple_bins", "0.1.0"), -// ("net2", "0.2.31"), -// ("nibble_vec", "0.0.3"), -// ("nix", "0.8.1"), -// ("num", "0.1.41"), -// ("num-bigint", "0.1.41"), -// ("num-complex", "0.1.41"), -// ("num-integer", "0.1.35"), -// ("num-iter", "0.1.34"), -// ("num-rational", "0.1.40"), -// ("num-traits", "0.1.41"), -// ("num_cpus", "1.8.0"), -// ("open", "1.2.1"), -// ("openssl", "0.9.23"), -// ("openssl-probe", "0.1.2"), -// ("openssl-sys", "0.9.24"), -// ("os_pipe", "0.5.1"), -// ("owning_ref", "0.3.3"), -// ("panic_abort", "0.0.0"), -// ("panic_unwind", "0.0.0"), -// ("parking_lot", "0.5.3"), -// ("parking_lot_core", "0.2.9"), -// ("percent-encoding", "1.0.1"), -// ("pest", "0.3.3"), -// ("pkg-config", "0.3.9"), -// ("proc_macro", "0.0.0"), -// ("profiler_builtins", "0.0.0"), -// ("pulldown-cmark", "0.0.15"), -// ("pulldown-cmark", "0.1.0"), -// ("quick-error", "1.2.1"), -// ("quine-mc_cluskey", "0.2.4"), -// ("quote", "0.3.15"), -// ("racer", "2.0.12"), -// ("radix_trie", "0.1.2"), -// ("rand", "0.3.20"), -// ("rayon", "0.9.0"), -// ("rayon-core", "1.3.0"), -// ("redox_syscall", "0.1.37"), -// ("redox_termios", "0.1.1"), -// ("reformat", "0.1.0"), -// ("reformat_with_range", "0.1.0"), -// ("regex", "0.1.80"), -// ("regex", "0.2.5"), -// ("regex-syntax", "0.3.9"), -// ("regex-syntax", "0.4.2"), -// ("remote-test-client", "0.1.0"), -// ("remote-test-server", "0.1.0"), -// ("rls", "0.125.0"), -// ("rls-analysis", "0.11.0"), -// ("rls-blacklist", "0.1.0"), -// ("rls-data", "0.15.0"), -// ("rls-rustc", "0.2.1"), -// ("rls-span", "0.4.0"), -// ("rls-vfs", "0.4.4"), -// ("rustbook", "0.1.0"), -// ("rustc", "0.0.0"), -// ("rustc-ap-rustc_cratesio_shim", "29.0.0"), -// ("rustc-ap-rustc_data_structures", "29.0.0"), -// ("rustc-ap-rustc_errors", "29.0.0"), -// ("rustc-ap-serialize", "29.0.0"), -// ("rustc-ap-syntax", "29.0.0"), -// ("rustc-ap-syntax_pos", "29.0.0"), -// ("rustc-demangle", "0.1.5"), -// ("rustc-main", "0.0.0"), -// ("rustc-serialize", "0.3.24"), -// ("rustc_allocator", "0.0.0"), -// ("rustc_apfloat", "0.0.0"), -// ("rustc_asan", "0.0.0"), -// ("rustc_back", "0.0.0"), -// ("rustc_binaryen", "0.0.0"), -// ("rustc_borrowck", "0.0.0"), -// ("rustc_const_eval", "0.0.0"), -// ("rustc_const_math", "0.0.0"), -// ("rustc_cratesio_shim", "0.0.0"), -// ("rustc_data_structures", "0.0.0"), -// ("rustc_driver", "0.0.0"), -// ("rustc_errors", "0.0.0"), -// ("rustc_incremental", "0.0.0"), -// ("rustc_lint", "0.0.0"), -// ("rustc_llvm", "0.0.0"), -// ("rustc_lsan", "0.0.0"), -// ("rustc_metadata", "0.0.0"), -// ("rustc_mir", "0.0.0"), -// ("rustc_msan", "0.0.0"), -// ("rustc_passes", "0.0.0"), -// ("rustc_platform_intrinsics", "0.0.0"), -// ("rustc_plugin", "0.0.0"), -// ("rustc_privacy", "0.0.0"), -// ("rustc_resolve", "0.0.0"), -// ("rustc_save_analysis", "0.0.0"), -// ("rustc_trans", "0.0.0"), -// ("rustc_trans_utils", "0.0.0"), -// ("rustc_tsan", "0.0.0"), -// ("rustc_typeck", "0.0.0"), -// ("rustdoc", "0.0.0"), -// ("rustdoc-themes", "0.1.0"), -// ("rustdoc-tool", "0.0.0"), -// ("rustfmt-nightly", "0.3.8"), -// ("same-file", "0.1.3"), -// ("same-file", "1.0.2"), -// ("schannel", "0.1.10"), -// ("scoped-tls", "0.1.0"), -// ("scopeguard", "0.1.2"), -// ("scopeguard", "0.3.3"), -// ("semver", "0.6.0"), -// ("semver", "0.8.0"), -// ("semver", "0.9.0"), -// ("semver-parser", "0.7.0"), -// ("serde", "1.0.27"), -// ("serde_derive", "1.0.27"), -// ("serde_derive_internals", "0.19.0"), -// ("serde_ignored", "0.0.4"), -// ("serde_json", "1.0.9"), -// ("serialize", "0.0.0"), -// ("shared_child", "0.2.1"), -// ("shell-escape", "0.1.3"), -// ("shlex", "0.1.1"), -// ("smallvec", "0.6.0"), -// ("socket2", "0.3.0"), -// ("stable_deref_trait", "1.0.0"), -// ("std", "0.0.0"), -// ("std_unicode", "0.0.0"), -// ("strsim", "0.6.0"), -// ("syn", "0.11.11"), -// ("synom", "0.11.3"), -// ("synstructure", "0.6.1"), -// ("syntax", "0.0.0"), -// ("syntax_ext", "0.0.0"), -// ("syntax_pos", "0.0.0"), -// ("syntex_errors", "0.52.0"), -// ("syntex_pos", "0.52.0"), -// ("syntex_syntax", "0.52.0"), -// ("tar", "0.4.14"), -// ("tempdir", "0.3.5"), -// ("term", "0.0.0"), -// ("term", "0.4.6"), -// ("termcolor", "0.3.3"), -// ("termion", "1.5.1"), -// ("test", "0.0.0"), -// ("textwrap", "0.9.0"), -// ("thread-id", "2.0.0"), -// ("thread_local", "0.2.7"), -// ("thread_local", "0.3.5"), -// ("tidy", "0.1.0"), -// ("time", "0.1.39"), -// ("toml", "0.2.1"), -// ("toml", "0.4.5"), -// ("toml-query", "0.6.0"), -// ("unicode-bidi", "0.3.4"), -// ("unicode-normalization", "0.1.5"), -// ("unicode-segmentation", "1.2.0"), -// ("unicode-width", "0.1.4"), -// ("unicode-xid", "0.0.3"), -// ("unicode-xid", "0.0.4"), -// ("unreachable", "1.0.0"), -// ("unstable-book-gen", "0.1.0"), -// ("unwind", "0.0.0"), -// ("url", "1.6.0"), -// ("url_serde", "0.2.0"), -// ("userenv-sys", "0.2.0"), -// ("utf8-ranges", "0.1.3"), -// ("utf8-ranges", "1.0.0"), -// ("vcpkg", "0.2.2"), -// ("vec_map", "0.8.0"), -// ("void", "1.0.2"), -// ("walkdir", "1.0.7"), -// ("walkdir", "2.0.1"), -// ("winapi", "0.2.8"), -// ("winapi", "0.3.4"), -// ("winapi-build", "0.1.1"), -// ("winapi-i686-pc-windows-gnu", "0.4.0"), -// ("winapi-x86_64-pc-windows-gnu", "0.4.0"), -// ("wincolor", "0.1.4"), -// ("workspace_symbol", "0.1.0"), -// ("ws2_32-sys", "0.2.1"), -// ("xattr", "0.1.11"), -// ("xz2", "0.1.4"), -// ("yaml-rust", "0.3.5"), +/// Which crates to check against the whitelist? +static WHITELIST_CRATES: &'static [Crate] = + &[Crate("rustc", "0.0.0"), Crate("rustc_trans", "0.0.0")]; + +/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. +static WHITELIST: &'static [Crate] = &[ +// Crate("ar", "0.3.1"), +// Crate("arena", "0.0.0"), +// Crate("backtrace", "0.3.5"), +// Crate("backtrace-sys", "0.1.16"), +// Crate("bitflags", "1.0.1"), +// Crate("build_helper", "0.1.0"), +// Crate("byteorder", "1.2.1"), +// Crate("cc", "1.0.4"), +// Crate("cfg-if", "0.1.2"), +// Crate("cmake", "0.1.29"), +// Crate("filetime", "0.1.15"), +// Crate("flate2", "1.0.1"), +// Crate("fmt_macros", "0.0.0"), +// Crate("fuchsia-zircon", "0.3.3"), +// Crate("fuchsia-zircon-sys", "0.3.3"), +// Crate("graphviz", "0.0.0"), +// Crate("jobserver", "0.1.9"), +// Crate("kernel32-sys", "0.2.2"), +// Crate("lazy_static", "0.2.11"), +// Crate("libc", "0.2.36"), +// Crate("log", "0.4.1"), +// Crate("log_settings", "0.1.1"), +// Crate("miniz-sys", "0.1.10"), +// Crate("num_cpus", "1.8.0"), +// Crate("owning_ref", "0.3.3"), +// Crate("parking_lot", "0.5.3"), +// Crate("parking_lot_core", "0.2.9"), +// Crate("rand", "0.3.20"), +// Crate("redox_syscall", "0.1.37"), +// Crate("rustc", "0.0.0"), +// Crate("rustc-demangle", "0.1.5"), +// Crate("rustc_allocator", "0.0.0"), +// Crate("rustc_apfloat", "0.0.0"), +// Crate("rustc_back", "0.0.0"), +// Crate("rustc_binaryen", "0.0.0"), +// Crate("rustc_const_eval", "0.0.0"), +// Crate("rustc_const_math", "0.0.0"), +// Crate("rustc_cratesio_shim", "0.0.0"), +// Crate("rustc_data_structures", "0.0.0"), +// Crate("rustc_errors", "0.0.0"), +// Crate("rustc_incremental", "0.0.0"), +// Crate("rustc_llvm", "0.0.0"), +// Crate("rustc_mir", "0.0.0"), +// Crate("rustc_platform_intrinsics", "0.0.0"), +// Crate("rustc_trans", "0.0.0"), +// Crate("rustc_trans_utils", "0.0.0"), +// Crate("serialize", "0.0.0"), +// Crate("smallvec", "0.6.0"), +// Crate("stable_deref_trait", "1.0.0"), +// Crate("syntax", "0.0.0"), +// Crate("syntax_pos", "0.0.0"), +// Crate("tempdir", "0.3.5"), +// Crate("unicode-width", "0.1.4"), +// Crate("winapi", "0.2.8"), +// Crate("winapi", "0.3.4"), +// Crate("winapi-build", "0.1.1"), +// Crate("winapi-i686-pc-windows-gnu", "0.4.0"), +// Crate("winapi-x86_64-pc-windows-gnu", "0.4.0"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... #[derive(Deserialize)] struct Output { - packages: Vec, + resolve: Resolve, // Not used, but needed to not confuse serde :P - #[allow(dead_code)] resolve: Resolve, -} - -#[derive(Deserialize)] -struct Package { - name: String, - version: String, - - // Not used, but needed to not confuse serde :P - #[allow(dead_code)] id: String, - #[allow(dead_code)] source: Option, - #[allow(dead_code)] manifest_path: String, + #[allow(dead_code)] packages: Vec, } // Not used, but needed to not confuse serde :P #[allow(dead_code)] +#[derive(Deserialize)] +struct Package { + name: String, + version: String, + id: String, + source: Option, + manifest_path: String, +} + #[derive(Deserialize)] struct Resolve { nodes: Vec, } -// Not used, but needed to not confuse serde :P -#[allow(dead_code)] #[derive(Deserialize)] struct ResolveNode { id: String, dependencies: Vec, } +/// A unique identifier for a crate +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] +struct Crate<'a>(&'a str, &'a str); // (name, version) + +impl<'a> Crate<'a> { + pub fn from_str(s: &'a str) -> Self { + let mut parts = s.split(" "); + let name = parts.next().unwrap(); + let version = parts.next().unwrap(); + + Crate(name, version) + } + + pub fn id_str(&self) -> String { + format!("{} {}", self.0, self.1) + } +} + /// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. /// /// Specifically, this checks that the license is correct. @@ -423,25 +192,27 @@ pub fn check(path: &Path, bad: &mut bool) { assert!(saw_dir, "no vendored source"); } -/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. +/// Checks the dependency of WHITELIST_CRATES at the given path. Changes `bad` to `true` if a check +/// failed. /// -/// Specifically, this checks that the dependencies are on the whitelist. +/// Specifically, this checks that the dependencies are on the WHITELIST. pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { - // Check dependencies - let deps: HashSet<_> = get_deps(&path, &cargo) - .into_iter() - .map(|Package { name, version, .. }| (name, version)) - .collect(); - let whitelist: HashSet<(String, String)> = WHITELIST - .iter() - .map(|&(n, v)| (n.to_owned(), v.to_owned())) - .collect(); + // Get dependencies from cargo metadata + let resolve = get_deps(path, cargo); - // Dependencies not in the whitelist - let mut unapproved: Vec<_> = deps.difference(&whitelist).collect(); + // Get the whitelist into a convenient form + let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect(); + + // Check dependencies + let mut unapproved = Vec::new(); + for &krate in WHITELIST_CRATES.iter() { + let mut bad = check_crate_whitelist(&whitelist, &resolve, krate); + unapproved.append(&mut bad); + } // For ease of reading - unapproved.sort(); + unapproved.sort_unstable(); + unapproved.dedup(); if unapproved.len() > 0 { println!("Dependencies not on the whitelist:"); @@ -492,7 +263,7 @@ fn extract_license(line: &str) -> String { } /// Get the dependencies of the crate at the given path using `cargo metadata`. -fn get_deps(path: &Path, cargo: &Path) -> Vec { +fn get_deps(path: &Path, cargo: &Path) -> Resolve { // Run `cargo metadata` to get the set of dependencies let output = Command::new(cargo) .arg("metadata") @@ -506,5 +277,41 @@ fn get_deps(path: &Path, cargo: &Path) -> Vec { let output = String::from_utf8_lossy(&output); let output: Output = serde_json::from_str(&output).unwrap(); - output.packages + output.resolve +} + +/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on +/// the whitelist. Returns a list of illegal dependencies. +fn check_crate_whitelist<'a>( + whitelist: &'a HashSet, + resolve: &'a Resolve, + krate: Crate<'a>, +) -> Vec> { + // Will contain bad deps + let mut unapproved = Vec::new(); + + // If this dependency is not on the WHITELIST, add to bad set + if !whitelist.contains(&krate) { + unapproved.push(krate); + } + + // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!) + let to_check = resolve + .nodes + .iter() + .find(|n| n.id.starts_with(&krate.id_str())) + .expect("crate does not exist"); + + for dep in to_check.dependencies.iter() { + let krate = Crate::from_str(dep); + let mut bad = check_crate_whitelist(whitelist, resolve, krate); + + unapproved.append(&mut bad); + } + + // Remove duplicates + unapproved.sort_unstable(); + unapproved.dedup(); + + unapproved } From e3a374ac7d1c166e963981aff3b59c37eb7270a6 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Tue, 27 Feb 2018 12:02:54 -0600 Subject: [PATCH 08/17] Fix alexcrichton's comments --- src/tools/tidy/src/deps.rs | 172 +++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 95 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 0d351c9e735..97cb8ed45b9 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -10,7 +10,7 @@ //! Check license of third-party deps by inspecting src/vendor -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; use std::fs::File; use std::io::Read; use std::path::Path; @@ -49,69 +49,65 @@ ]; /// Which crates to check against the whitelist? -static WHITELIST_CRATES: &'static [Crate] = - &[Crate("rustc", "0.0.0"), Crate("rustc_trans", "0.0.0")]; +static WHITELIST_CRATES: &'static [Crate] = &[Crate("rustc"), Crate("rustc_trans")]; /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ -// Crate("ar", "0.3.1"), -// Crate("arena", "0.0.0"), -// Crate("backtrace", "0.3.5"), -// Crate("backtrace-sys", "0.1.16"), -// Crate("bitflags", "1.0.1"), -// Crate("build_helper", "0.1.0"), -// Crate("byteorder", "1.2.1"), -// Crate("cc", "1.0.4"), -// Crate("cfg-if", "0.1.2"), -// Crate("cmake", "0.1.29"), -// Crate("filetime", "0.1.15"), -// Crate("flate2", "1.0.1"), -// Crate("fmt_macros", "0.0.0"), -// Crate("fuchsia-zircon", "0.3.3"), -// Crate("fuchsia-zircon-sys", "0.3.3"), -// Crate("graphviz", "0.0.0"), -// Crate("jobserver", "0.1.9"), -// Crate("kernel32-sys", "0.2.2"), -// Crate("lazy_static", "0.2.11"), -// Crate("libc", "0.2.36"), -// Crate("log", "0.4.1"), -// Crate("log_settings", "0.1.1"), -// Crate("miniz-sys", "0.1.10"), -// Crate("num_cpus", "1.8.0"), -// Crate("owning_ref", "0.3.3"), -// Crate("parking_lot", "0.5.3"), -// Crate("parking_lot_core", "0.2.9"), -// Crate("rand", "0.3.20"), -// Crate("redox_syscall", "0.1.37"), -// Crate("rustc", "0.0.0"), -// Crate("rustc-demangle", "0.1.5"), -// Crate("rustc_allocator", "0.0.0"), -// Crate("rustc_apfloat", "0.0.0"), -// Crate("rustc_back", "0.0.0"), -// Crate("rustc_binaryen", "0.0.0"), -// Crate("rustc_const_eval", "0.0.0"), -// Crate("rustc_const_math", "0.0.0"), -// Crate("rustc_cratesio_shim", "0.0.0"), -// Crate("rustc_data_structures", "0.0.0"), -// Crate("rustc_errors", "0.0.0"), -// Crate("rustc_incremental", "0.0.0"), -// Crate("rustc_llvm", "0.0.0"), -// Crate("rustc_mir", "0.0.0"), -// Crate("rustc_platform_intrinsics", "0.0.0"), -// Crate("rustc_trans", "0.0.0"), -// Crate("rustc_trans_utils", "0.0.0"), -// Crate("serialize", "0.0.0"), -// Crate("smallvec", "0.6.0"), -// Crate("stable_deref_trait", "1.0.0"), -// Crate("syntax", "0.0.0"), -// Crate("syntax_pos", "0.0.0"), -// Crate("tempdir", "0.3.5"), -// Crate("unicode-width", "0.1.4"), -// Crate("winapi", "0.2.8"), -// Crate("winapi", "0.3.4"), -// Crate("winapi-build", "0.1.1"), -// Crate("winapi-i686-pc-windows-gnu", "0.4.0"), -// Crate("winapi-x86_64-pc-windows-gnu", "0.4.0"), +// Crate("ar "), +// Crate("arena "), +// Crate("backtrace "), +// Crate("backtrace-sys "), +// Crate("bitflags "), +// Crate("build_helper "), +// Crate("byteorder "), +// Crate("cc "), +// Crate("cfg-if "), +// Crate("cmake "), +// Crate("filetime "), +// Crate("flate2 "), +// Crate("fmt_macros "), +// Crate("fuchsia-zircon "), +// Crate("fuchsia-zircon-sys "), +// Crate("graphviz "), +// Crate("jobserver "), +// Crate("kernel32-sys "), +// Crate("lazy_static "), +// Crate("libc "), +// Crate("log "), +// Crate("log_settings "), +// Crate("miniz-sys "), +// Crate("num_cpus "), +// Crate("owning_ref "), +// Crate("parking_lot "), +// Crate("parking_lot_core "), +// Crate("rand "), +// Crate("redox_syscall "), +// Crate("rustc "), +// Crate("rustc-demangle "), +// Crate("rustc_allocator "), +// Crate("rustc_apfloat "), +// Crate("rustc_back "), +// Crate("rustc_binaryen "), +// Crate("rustc_const_eval "), +// Crate("rustc_const_math "), +// Crate("rustc_cratesio_shim "), +// Crate("rustc_data_structures "), +// Crate("rustc_errors "), +// Crate("rustc_incremental "), +// Crate("rustc_llvm "), +// Crate("rustc_mir "), +// Crate("rustc_platform_intrinsics "), +// Crate("rustc_trans "), +// Crate("rustc_trans_utils "), +// Crate("serialize "), +// Crate("smallvec "), +// Crate("stable_deref_trait "), +// Crate("syntax "), +// Crate("syntax_pos "), +// Crate("tempdir "), +// Crate("unicode-width "), +// Crate("winapi "), +// Crate("winapi-build"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... @@ -119,20 +115,6 @@ #[derive(Deserialize)] struct Output { resolve: Resolve, - - // Not used, but needed to not confuse serde :P - #[allow(dead_code)] packages: Vec, -} - -// Not used, but needed to not confuse serde :P -#[allow(dead_code)] -#[derive(Deserialize)] -struct Package { - name: String, - version: String, - id: String, - source: Option, - manifest_path: String, } #[derive(Deserialize)] @@ -148,19 +130,18 @@ struct ResolveNode { /// A unique identifier for a crate #[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] -struct Crate<'a>(&'a str, &'a str); // (name, version) +struct Crate<'a>(&'a str); // (name,) impl<'a> Crate<'a> { pub fn from_str(s: &'a str) -> Self { let mut parts = s.split(" "); let name = parts.next().unwrap(); - let version = parts.next().unwrap(); - Crate(name, version) + Crate(name) } pub fn id_str(&self) -> String { - format!("{} {}", self.0, self.1) + format!("{} ", self.0) } } @@ -204,20 +185,17 @@ pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect(); // Check dependencies - let mut unapproved = Vec::new(); + let mut visited = BTreeSet::new(); + let mut unapproved = BTreeSet::new(); for &krate in WHITELIST_CRATES.iter() { - let mut bad = check_crate_whitelist(&whitelist, &resolve, krate); + let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate); unapproved.append(&mut bad); } - // For ease of reading - unapproved.sort_unstable(); - unapproved.dedup(); - if unapproved.len() > 0 { println!("Dependencies not on the whitelist:"); for dep in unapproved { - println!("* {} {}", dep.0, dep.1); // name version + println!("* {}", dep.id_str()); } *bad = true; } @@ -282,17 +260,25 @@ fn get_deps(path: &Path, cargo: &Path) -> Resolve { /// Checks the dependencies of the given crate from the given cargo metadata to see if they are on /// the whitelist. Returns a list of illegal dependencies. -fn check_crate_whitelist<'a>( +fn check_crate_whitelist<'a, 'b>( whitelist: &'a HashSet, resolve: &'a Resolve, + visited: &'b mut BTreeSet>, krate: Crate<'a>, -) -> Vec> { +) -> BTreeSet> { // Will contain bad deps - let mut unapproved = Vec::new(); + let mut unapproved = BTreeSet::new(); + + // Check if we have already visited this crate + if visited.contains(&krate) { + return unapproved; + } + + visited.insert(krate); // If this dependency is not on the WHITELIST, add to bad set if !whitelist.contains(&krate) { - unapproved.push(krate); + unapproved.insert(krate); } // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!) @@ -304,14 +290,10 @@ fn check_crate_whitelist<'a>( for dep in to_check.dependencies.iter() { let krate = Crate::from_str(dep); - let mut bad = check_crate_whitelist(whitelist, resolve, krate); + let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate); unapproved.append(&mut bad); } - // Remove duplicates - unapproved.sort_unstable(); - unapproved.dedup(); - unapproved } From f89abd63ca10662439692dd085a52b5757bc3259 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Tue, 27 Feb 2018 12:47:49 -0600 Subject: [PATCH 09/17] uncomment whitelist --- src/tools/tidy/src/deps.rs | 110 ++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 97cb8ed45b9..cd089b70d77 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -53,61 +53,61 @@ /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ -// Crate("ar "), -// Crate("arena "), -// Crate("backtrace "), -// Crate("backtrace-sys "), -// Crate("bitflags "), -// Crate("build_helper "), -// Crate("byteorder "), -// Crate("cc "), -// Crate("cfg-if "), -// Crate("cmake "), -// Crate("filetime "), -// Crate("flate2 "), -// Crate("fmt_macros "), -// Crate("fuchsia-zircon "), -// Crate("fuchsia-zircon-sys "), -// Crate("graphviz "), -// Crate("jobserver "), -// Crate("kernel32-sys "), -// Crate("lazy_static "), -// Crate("libc "), -// Crate("log "), -// Crate("log_settings "), -// Crate("miniz-sys "), -// Crate("num_cpus "), -// Crate("owning_ref "), -// Crate("parking_lot "), -// Crate("parking_lot_core "), -// Crate("rand "), -// Crate("redox_syscall "), -// Crate("rustc "), -// Crate("rustc-demangle "), -// Crate("rustc_allocator "), -// Crate("rustc_apfloat "), -// Crate("rustc_back "), -// Crate("rustc_binaryen "), -// Crate("rustc_const_eval "), -// Crate("rustc_const_math "), -// Crate("rustc_cratesio_shim "), -// Crate("rustc_data_structures "), -// Crate("rustc_errors "), -// Crate("rustc_incremental "), -// Crate("rustc_llvm "), -// Crate("rustc_mir "), -// Crate("rustc_platform_intrinsics "), -// Crate("rustc_trans "), -// Crate("rustc_trans_utils "), -// Crate("serialize "), -// Crate("smallvec "), -// Crate("stable_deref_trait "), -// Crate("syntax "), -// Crate("syntax_pos "), -// Crate("tempdir "), -// Crate("unicode-width "), -// Crate("winapi "), -// Crate("winapi-build"), + Crate("ar "), + Crate("arena "), + Crate("backtrace "), + Crate("backtrace-sys "), + Crate("bitflags "), + Crate("build_helper "), + Crate("byteorder "), + Crate("cc "), + Crate("cfg-if "), + Crate("cmake "), + Crate("filetime "), + Crate("flate2 "), + Crate("fmt_macros "), + Crate("fuchsia-zircon "), + Crate("fuchsia-zircon-sys "), + Crate("graphviz "), + Crate("jobserver "), + Crate("kernel32-sys "), + Crate("lazy_static "), + Crate("libc "), + Crate("log "), + Crate("log_settings "), + Crate("miniz-sys "), + Crate("num_cpus "), + Crate("owning_ref "), + Crate("parking_lot "), + Crate("parking_lot_core "), + Crate("rand "), + Crate("redox_syscall "), + Crate("rustc "), + Crate("rustc-demangle "), + Crate("rustc_allocator "), + Crate("rustc_apfloat "), + Crate("rustc_back "), + Crate("rustc_binaryen "), + Crate("rustc_const_eval "), + Crate("rustc_const_math "), + Crate("rustc_cratesio_shim "), + Crate("rustc_data_structures "), + Crate("rustc_errors "), + Crate("rustc_incremental "), + Crate("rustc_llvm "), + Crate("rustc_mir "), + Crate("rustc_platform_intrinsics "), + Crate("rustc_trans "), + Crate("rustc_trans_utils "), + Crate("serialize "), + Crate("smallvec "), + Crate("stable_deref_trait "), + Crate("syntax "), + Crate("syntax_pos "), + Crate("tempdir "), + Crate("unicode-width "), + Crate("winapi "), + Crate("winapi-build"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... From 489f2f1206fbae4da67bdcbae70cd896f0537a46 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Tue, 27 Feb 2018 12:50:34 -0600 Subject: [PATCH 10/17] Remove spurious whitespace --- src/tools/tidy/src/deps.rs | 108 ++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index cd089b70d77..8cfba21eb6d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -53,60 +53,60 @@ /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ - Crate("ar "), - Crate("arena "), - Crate("backtrace "), - Crate("backtrace-sys "), - Crate("bitflags "), - Crate("build_helper "), - Crate("byteorder "), - Crate("cc "), - Crate("cfg-if "), - Crate("cmake "), - Crate("filetime "), - Crate("flate2 "), - Crate("fmt_macros "), - Crate("fuchsia-zircon "), - Crate("fuchsia-zircon-sys "), - Crate("graphviz "), - Crate("jobserver "), - Crate("kernel32-sys "), - Crate("lazy_static "), - Crate("libc "), - Crate("log "), - Crate("log_settings "), - Crate("miniz-sys "), - Crate("num_cpus "), - Crate("owning_ref "), - Crate("parking_lot "), - Crate("parking_lot_core "), - Crate("rand "), - Crate("redox_syscall "), - Crate("rustc "), - Crate("rustc-demangle "), - Crate("rustc_allocator "), - Crate("rustc_apfloat "), - Crate("rustc_back "), - Crate("rustc_binaryen "), - Crate("rustc_const_eval "), - Crate("rustc_const_math "), - Crate("rustc_cratesio_shim "), - Crate("rustc_data_structures "), - Crate("rustc_errors "), - Crate("rustc_incremental "), - Crate("rustc_llvm "), - Crate("rustc_mir "), - Crate("rustc_platform_intrinsics "), - Crate("rustc_trans "), - Crate("rustc_trans_utils "), - Crate("serialize "), - Crate("smallvec "), - Crate("stable_deref_trait "), - Crate("syntax "), - Crate("syntax_pos "), - Crate("tempdir "), - Crate("unicode-width "), - Crate("winapi "), + Crate("ar"), + Crate("arena"), + Crate("backtrace"), + Crate("backtrace-sys"), + Crate("bitflags"), + Crate("build_helper"), + Crate("byteorder"), + Crate("cc"), + Crate("cfg-if"), + Crate("cmake"), + Crate("filetime"), + Crate("flate2"), + Crate("fmt_macros"), + Crate("fuchsia-zircon"), + Crate("fuchsia-zircon-sys"), + Crate("graphviz"), + Crate("jobserver"), + Crate("kernel32-sys"), + Crate("lazy_static"), + Crate("libc"), + Crate("log"), + Crate("log_settings"), + Crate("miniz-sys"), + Crate("num_cpus"), + Crate("owning_ref"), + Crate("parking_lot"), + Crate("parking_lot_core"), + Crate("rand"), + Crate("redox_syscall"), + Crate("rustc"), + Crate("rustc-demangle"), + Crate("rustc_allocator"), + Crate("rustc_apfloat"), + Crate("rustc_back"), + Crate("rustc_binaryen"), + Crate("rustc_const_eval"), + Crate("rustc_const_math"), + Crate("rustc_cratesio_shim"), + Crate("rustc_data_structures"), + Crate("rustc_errors"), + Crate("rustc_incremental"), + Crate("rustc_llvm"), + Crate("rustc_mir"), + Crate("rustc_platform_intrinsics"), + Crate("rustc_trans"), + Crate("rustc_trans_utils"), + Crate("serialize"), + Crate("smallvec"), + Crate("stable_deref_trait"), + Crate("syntax"), + Crate("syntax_pos"), + Crate("tempdir"), + Crate("unicode-width"), + Crate("winapi"), Crate("winapi-build"), ]; From 0fc5daebfb4218312f0f2ee28de7c6bcd40b632c Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Tue, 27 Feb 2018 15:56:16 -0600 Subject: [PATCH 11/17] Add a few missing deps --- src/tools/tidy/src/deps.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 8cfba21eb6d..d63926ee3bd 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -108,6 +108,9 @@ Crate("unicode-width"), Crate("winapi"), Crate("winapi-build"), + Crate("proc_macro"), + Crate("winapi-i686-pc-windows-gnu"), + Crate("winapi-x86_64-pc-windows-gnu"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... From 18f0533c4cf5e320df1f2e8724f98f550ca65130 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Tue, 27 Feb 2018 18:33:55 -0600 Subject: [PATCH 12/17] different versions may have different deps --- src/tools/tidy/src/deps.rs | 150 +++++++++++++++++++++---------------- 1 file changed, 84 insertions(+), 66 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index d63926ee3bd..f0801b806ec 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -49,68 +49,70 @@ ]; /// Which crates to check against the whitelist? -static WHITELIST_CRATES: &'static [Crate] = &[Crate("rustc"), Crate("rustc_trans")]; +static WHITELIST_CRATES: &'static [CrateVersion] = &[ + CrateVersion("rustc", "0.0.0"), + CrateVersion("rustc_trans", "0.0.0"), +]; /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ - Crate("ar"), - Crate("arena"), - Crate("backtrace"), - Crate("backtrace-sys"), - Crate("bitflags"), - Crate("build_helper"), - Crate("byteorder"), - Crate("cc"), - Crate("cfg-if"), - Crate("cmake"), - Crate("filetime"), - Crate("flate2"), - Crate("fmt_macros"), - Crate("fuchsia-zircon"), - Crate("fuchsia-zircon-sys"), - Crate("graphviz"), - Crate("jobserver"), - Crate("kernel32-sys"), - Crate("lazy_static"), - Crate("libc"), - Crate("log"), - Crate("log_settings"), - Crate("miniz-sys"), - Crate("num_cpus"), - Crate("owning_ref"), - Crate("parking_lot"), - Crate("parking_lot_core"), - Crate("rand"), - Crate("redox_syscall"), - Crate("rustc"), - Crate("rustc-demangle"), - Crate("rustc_allocator"), - Crate("rustc_apfloat"), - Crate("rustc_back"), - Crate("rustc_binaryen"), - Crate("rustc_const_eval"), - Crate("rustc_const_math"), - Crate("rustc_cratesio_shim"), - Crate("rustc_data_structures"), - Crate("rustc_errors"), - Crate("rustc_incremental"), - Crate("rustc_llvm"), - Crate("rustc_mir"), - Crate("rustc_platform_intrinsics"), - Crate("rustc_trans"), - Crate("rustc_trans_utils"), - Crate("serialize"), - Crate("smallvec"), - Crate("stable_deref_trait"), - Crate("syntax"), - Crate("syntax_pos"), - Crate("tempdir"), - Crate("unicode-width"), - Crate("winapi"), - Crate("winapi-build"), - Crate("proc_macro"), - Crate("winapi-i686-pc-windows-gnu"), - Crate("winapi-x86_64-pc-windows-gnu"), +// Crate("ar"), +// Crate("arena"), +// Crate("backtrace"), +// Crate("backtrace-sys"), +// Crate("bitflags"), +// Crate("build_helper"), +// Crate("byteorder"), +// Crate("cc"), +// Crate("cfg-if"), +// Crate("cmake"), +// Crate("filetime"), +// Crate("flate2"), +// Crate("fmt_macros"), +// Crate("fuchsia-zircon"), +// Crate("fuchsia-zircon-sys"), +// Crate("graphviz"), +// Crate("jobserver"), +// Crate("kernel32-sys"), +// Crate("lazy_static"), +// Crate("libc"), +// Crate("log"), +// Crate("log_settings"), +// Crate("miniz-sys"), +// Crate("num_cpus"), +// Crate("owning_ref"), +// Crate("parking_lot"), +// Crate("parking_lot_core"), +// Crate("rand"), +// Crate("redox_syscall"), +// Crate("rustc"), +// Crate("rustc-demangle"), +// Crate("rustc_allocator"), +// Crate("rustc_apfloat"), +// Crate("rustc_back"), +// Crate("rustc_binaryen"), +// Crate("rustc_const_eval"), +// Crate("rustc_const_math"), +// Crate("rustc_cratesio_shim"), +// Crate("rustc_data_structures"), +// Crate("rustc_errors"), +// Crate("rustc_incremental"), +// Crate("rustc_llvm"), +// Crate("rustc_mir"), +// Crate("rustc_platform_intrinsics"), +// Crate("rustc_trans"), +// Crate("rustc_trans_utils"), +// Crate("serialize"), +// Crate("smallvec"), +// Crate("stable_deref_trait"), +// Crate("syntax"), +// Crate("syntax_pos"), +// Crate("tempdir"), +// Crate("unicode-width"), +// Crate("winapi"), +// Crate("winapi-build"), +// Crate("winapi-i686-pc-windows-gnu"), +// Crate("winapi-x86_64-pc-windows-gnu"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... @@ -135,16 +137,32 @@ struct ResolveNode { #[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] struct Crate<'a>(&'a str); // (name,) +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] +struct CrateVersion<'a>(&'a str, &'a str); // (name, version) + impl<'a> Crate<'a> { + pub fn id_str(&self) -> String { + format!("{} ", self.0) + } +} + +impl<'a> CrateVersion<'a> { pub fn from_str(s: &'a str) -> Self { let mut parts = s.split(" "); let name = parts.next().unwrap(); + let version = parts.next().unwrap(); - Crate(name) + CrateVersion(name, version) } pub fn id_str(&self) -> String { - format!("{} ", self.0) + format!("{} {}", self.0, self.1) + } +} + +impl<'a> From> for Crate<'a> { + fn from(cv: CrateVersion<'a>) -> Crate<'a> { + Crate(cv.0) } } @@ -266,8 +284,8 @@ fn get_deps(path: &Path, cargo: &Path) -> Resolve { fn check_crate_whitelist<'a, 'b>( whitelist: &'a HashSet, resolve: &'a Resolve, - visited: &'b mut BTreeSet>, - krate: Crate<'a>, + visited: &'b mut BTreeSet>, + krate: CrateVersion<'a>, ) -> BTreeSet> { // Will contain bad deps let mut unapproved = BTreeSet::new(); @@ -280,8 +298,8 @@ fn check_crate_whitelist<'a, 'b>( visited.insert(krate); // If this dependency is not on the WHITELIST, add to bad set - if !whitelist.contains(&krate) { - unapproved.insert(krate); + if !whitelist.contains(&krate.into()) { + unapproved.insert(krate.into()); } // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!) @@ -292,7 +310,7 @@ fn check_crate_whitelist<'a, 'b>( .expect("crate does not exist"); for dep in to_check.dependencies.iter() { - let krate = Crate::from_str(dep); + let krate = CrateVersion::from_str(dep); let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate); unapproved.append(&mut bad); From 1fca9e0430bba507d21b5373063d850ffe6d550b Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Wed, 28 Feb 2018 14:12:15 -0600 Subject: [PATCH 13/17] Don't check in-tree deps --- src/tools/tidy/src/deps.rs | 51 +++++++++----------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index f0801b806ec..f22f8ac8efc 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -56,61 +56,27 @@ /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ -// Crate("ar"), -// Crate("arena"), // Crate("backtrace"), // Crate("backtrace-sys"), // Crate("bitflags"), -// Crate("build_helper"), // Crate("byteorder"), // Crate("cc"), // Crate("cfg-if"), -// Crate("cmake"), -// Crate("filetime"), // Crate("flate2"), -// Crate("fmt_macros"), // Crate("fuchsia-zircon"), // Crate("fuchsia-zircon-sys"), -// Crate("graphviz"), // Crate("jobserver"), -// Crate("kernel32-sys"), // Crate("lazy_static"), // Crate("libc"), // Crate("log"), -// Crate("log_settings"), // Crate("miniz-sys"), // Crate("num_cpus"), -// Crate("owning_ref"), -// Crate("parking_lot"), -// Crate("parking_lot_core"), // Crate("rand"), -// Crate("redox_syscall"), // Crate("rustc"), // Crate("rustc-demangle"), -// Crate("rustc_allocator"), -// Crate("rustc_apfloat"), -// Crate("rustc_back"), -// Crate("rustc_binaryen"), -// Crate("rustc_const_eval"), -// Crate("rustc_const_math"), -// Crate("rustc_cratesio_shim"), -// Crate("rustc_data_structures"), -// Crate("rustc_errors"), -// Crate("rustc_incremental"), -// Crate("rustc_llvm"), -// Crate("rustc_mir"), -// Crate("rustc_platform_intrinsics"), // Crate("rustc_trans"), -// Crate("rustc_trans_utils"), -// Crate("serialize"), -// Crate("smallvec"), -// Crate("stable_deref_trait"), -// Crate("syntax"), -// Crate("syntax_pos"), // Crate("tempdir"), -// Crate("unicode-width"), // Crate("winapi"), -// Crate("winapi-build"), // Crate("winapi-i686-pc-windows-gnu"), // Crate("winapi-x86_64-pc-windows-gnu"), ]; @@ -147,12 +113,16 @@ pub fn id_str(&self) -> String { } impl<'a> CrateVersion<'a> { - pub fn from_str(s: &'a str) -> Self { + /// Returns the struct and whether or not the dep is in-tree + pub fn from_str(s: &'a str) -> (Self, bool) { let mut parts = s.split(" "); let name = parts.next().unwrap(); let version = parts.next().unwrap(); + let path = parts.next().unwrap(); - CrateVersion(name, version) + let is_path_dep = path.starts_with("(path+"); + + (CrateVersion(name, version), is_path_dep) } pub fn id_str(&self) -> String { @@ -310,10 +280,13 @@ fn check_crate_whitelist<'a, 'b>( .expect("crate does not exist"); for dep in to_check.dependencies.iter() { - let krate = CrateVersion::from_str(dep); - let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate); + let (krate, is_path_dep) = CrateVersion::from_str(dep); - unapproved.append(&mut bad); + // We don't check in-tree deps + if !is_path_dep { + let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate); + unapproved.append(&mut bad); + } } unapproved From 6180a3f7da1fa8693344e96049f7400cd0d04088 Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Wed, 28 Feb 2018 14:25:34 -0600 Subject: [PATCH 14/17] enable whitelist --- src/tools/tidy/src/deps.rs | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index f22f8ac8efc..4a76e2aaa16 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -56,29 +56,29 @@ /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ -// Crate("backtrace"), -// Crate("backtrace-sys"), -// Crate("bitflags"), -// Crate("byteorder"), -// Crate("cc"), -// Crate("cfg-if"), -// Crate("flate2"), -// Crate("fuchsia-zircon"), -// Crate("fuchsia-zircon-sys"), -// Crate("jobserver"), -// Crate("lazy_static"), -// Crate("libc"), -// Crate("log"), -// Crate("miniz-sys"), -// Crate("num_cpus"), -// Crate("rand"), -// Crate("rustc"), -// Crate("rustc-demangle"), -// Crate("rustc_trans"), -// Crate("tempdir"), -// Crate("winapi"), -// Crate("winapi-i686-pc-windows-gnu"), -// Crate("winapi-x86_64-pc-windows-gnu"), + Crate("backtrace"), + Crate("backtrace-sys"), + Crate("bitflags"), + Crate("byteorder"), + Crate("cc"), + Crate("cfg-if"), + Crate("flate2"), + Crate("fuchsia-zircon"), + Crate("fuchsia-zircon-sys"), + Crate("jobserver"), + Crate("lazy_static"), + Crate("libc"), + Crate("log"), + Crate("miniz-sys"), + Crate("num_cpus"), + Crate("rand"), + Crate("rustc"), + Crate("rustc-demangle"), + Crate("rustc_trans"), + Crate("tempdir"), + Crate("winapi"), + Crate("winapi-i686-pc-windows-gnu"), + Crate("winapi-x86_64-pc-windows-gnu"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... From bb01f3126d374f1448cfa3e19f2a1ca3bec8e8ce Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Wed, 28 Feb 2018 16:28:30 -0600 Subject: [PATCH 15/17] Do check the deps of path deps --- src/tools/tidy/src/deps.rs | 76 ++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 4a76e2aaa16..0f3b09c2978 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -56,29 +56,40 @@ /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ - Crate("backtrace"), - Crate("backtrace-sys"), - Crate("bitflags"), - Crate("byteorder"), - Crate("cc"), - Crate("cfg-if"), - Crate("flate2"), - Crate("fuchsia-zircon"), - Crate("fuchsia-zircon-sys"), - Crate("jobserver"), - Crate("lazy_static"), - Crate("libc"), - Crate("log"), - Crate("miniz-sys"), - Crate("num_cpus"), - Crate("rand"), - Crate("rustc"), - Crate("rustc-demangle"), - Crate("rustc_trans"), - Crate("tempdir"), - Crate("winapi"), - Crate("winapi-i686-pc-windows-gnu"), - Crate("winapi-x86_64-pc-windows-gnu"), +// Crate("ar"), +// Crate("backtrace"), +// Crate("backtrace-sys"), +// Crate("bitflags"), +// Crate("byteorder"), +// Crate("cc"), +// Crate("cfg-if"), +// Crate("cmake"), +// Crate("filetime"), +// Crate("flate2"), +// Crate("fuchsia-zircon"), +// Crate("fuchsia-zircon-sys"), +// Crate("jobserver"), +// Crate("kernel32-sys"), +// Crate("lazy_static"), +// Crate("libc"), +// Crate("log"), +// Crate("log_settings"), +// Crate("miniz-sys"), +// Crate("num_cpus"), +// Crate("owning_ref"), +// Crate("parking_lot"), +// Crate("parking_lot_core"), +// Crate("rand"), +// Crate("redox_syscall"), +// Crate("rustc-demangle"), +// Crate("smallvec"), +// Crate("stable_deref_trait"), +// Crate("tempdir"), +// Crate("unicode-width"), +// Crate("winapi"), +// Crate("winapi-build"), +// Crate("winapi-i686-pc-windows-gnu"), +// Crate("winapi-x86_64-pc-windows-gnu"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... @@ -179,7 +190,7 @@ pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { let mut visited = BTreeSet::new(); let mut unapproved = BTreeSet::new(); for &krate in WHITELIST_CRATES.iter() { - let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate); + let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false); unapproved.append(&mut bad); } @@ -256,6 +267,7 @@ fn check_crate_whitelist<'a, 'b>( resolve: &'a Resolve, visited: &'b mut BTreeSet>, krate: CrateVersion<'a>, + must_be_on_whitelist: bool, ) -> BTreeSet> { // Will contain bad deps let mut unapproved = BTreeSet::new(); @@ -267,9 +279,12 @@ fn check_crate_whitelist<'a, 'b>( visited.insert(krate); - // If this dependency is not on the WHITELIST, add to bad set - if !whitelist.contains(&krate.into()) { - unapproved.insert(krate.into()); + // If this path is in-tree, we don't require it to be on the whitelist + if must_be_on_whitelist { + // If this dependency is not on the WHITELIST, add to bad set + if !whitelist.contains(&krate.into()) { + unapproved.insert(krate.into()); + } } // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!) @@ -282,11 +297,8 @@ fn check_crate_whitelist<'a, 'b>( for dep in to_check.dependencies.iter() { let (krate, is_path_dep) = CrateVersion::from_str(dep); - // We don't check in-tree deps - if !is_path_dep { - let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate); - unapproved.append(&mut bad); - } + let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep); + unapproved.append(&mut bad); } unapproved From 1423da8b9a62f6d9d9e90396a4c1f471ceb694d0 Mon Sep 17 00:00:00 2001 From: Who? Me?! Date: Thu, 1 Mar 2018 21:22:06 -0600 Subject: [PATCH 16/17] Uncomment whitelist --- src/tools/tidy/src/deps.rs | 68 +++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 0f3b09c2978..aa46c6b7c4d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -56,40 +56,40 @@ /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. static WHITELIST: &'static [Crate] = &[ -// Crate("ar"), -// Crate("backtrace"), -// Crate("backtrace-sys"), -// Crate("bitflags"), -// Crate("byteorder"), -// Crate("cc"), -// Crate("cfg-if"), -// Crate("cmake"), -// Crate("filetime"), -// Crate("flate2"), -// Crate("fuchsia-zircon"), -// Crate("fuchsia-zircon-sys"), -// Crate("jobserver"), -// Crate("kernel32-sys"), -// Crate("lazy_static"), -// Crate("libc"), -// Crate("log"), -// Crate("log_settings"), -// Crate("miniz-sys"), -// Crate("num_cpus"), -// Crate("owning_ref"), -// Crate("parking_lot"), -// Crate("parking_lot_core"), -// Crate("rand"), -// Crate("redox_syscall"), -// Crate("rustc-demangle"), -// Crate("smallvec"), -// Crate("stable_deref_trait"), -// Crate("tempdir"), -// Crate("unicode-width"), -// Crate("winapi"), -// Crate("winapi-build"), -// Crate("winapi-i686-pc-windows-gnu"), -// Crate("winapi-x86_64-pc-windows-gnu"), + Crate("ar"), + Crate("backtrace"), + Crate("backtrace-sys"), + Crate("bitflags"), + Crate("byteorder"), + Crate("cc"), + Crate("cfg-if"), + Crate("cmake"), + Crate("filetime"), + Crate("flate2"), + Crate("fuchsia-zircon"), + Crate("fuchsia-zircon-sys"), + Crate("jobserver"), + Crate("kernel32-sys"), + Crate("lazy_static"), + Crate("libc"), + Crate("log"), + Crate("log_settings"), + Crate("miniz-sys"), + Crate("num_cpus"), + Crate("owning_ref"), + Crate("parking_lot"), + Crate("parking_lot_core"), + Crate("rand"), + Crate("redox_syscall"), + Crate("rustc-demangle"), + Crate("smallvec"), + Crate("stable_deref_trait"), + Crate("tempdir"), + Crate("unicode-width"), + Crate("winapi"), + Crate("winapi-build"), + Crate("winapi-i686-pc-windows-gnu"), + Crate("winapi-x86_64-pc-windows-gnu"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... From e5d292013be6a0074d185e0de4c563545a4ed755 Mon Sep 17 00:00:00 2001 From: Who? Me?! Date: Fri, 2 Mar 2018 12:15:02 -0600 Subject: [PATCH 17/17] Add ena to whitelist --- src/tools/tidy/src/deps.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index aa46c6b7c4d..f40c7a72a45 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -64,6 +64,7 @@ Crate("cc"), Crate("cfg-if"), Crate("cmake"), + Crate("ena"), Crate("filetime"), Crate("flate2"), Crate("fuchsia-zircon"),