Auto merge of #48456 - mark-i-m:whitelist, r=alexcrichton

Whitelist rustc dependencies

As per https://internals.rust-lang.org/t/rustc-dependency-policy/6537/8?u=nikomatsakis

cc @alexcrichton @nikomatsakis

src/Cargo.lock

@@ -2585,6 +2585,11 @@ dependencies = [
 [[package]]
 name = "tidy"
 version = "0.1.0"
+dependencies = [
+ "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "time"

src/bootstrap/test.rs

@@ -528,6 +528,7 @@ fn run(self, builder: &Builder) {
         println!("tidy check ({})", host);
         let mut cmd = builder.tool_cmd(Tool::Tidy);
         cmd.arg(build.src.join("src"));
+        cmd.arg(&build.initial_cargo);
         if !build.config.vendor {
             cmd.arg("--no-vendor");
         }

src/tools/tidy/Cargo.toml

@@ -2,3 +2,8 @@
 name = "tidy"
 version = "0.1.0"
 authors = ["Alex Crichton <alex@alexcrichton.com>"]
+
+[dependencies]
+serde = "1.0.8"
+serde_derive = "1.0.8"
+serde_json = "1.0.2"

src/tools/tidy/src/deps.rs

@@ -10,9 +10,13 @@
 
 //! Check license of third-party deps by inspecting src/vendor
 
+use std::collections::{BTreeSet, HashSet};
 use std::fs::File;
 use std::io::Read;
 use std::path::Path;
+use std::process::Command;
+
+use serde_json;
 
 static LICENSES: &'static [&'static str] = &[
     "MIT/Apache-2.0",
@@ -24,52 +28,182 @@
     "Unlicense/MIT",
 ];
 
-// These are exceptions to Rust's permissive licensing policy, and
-// should be considered bugs. Exceptions are only allowed in Rust
-// tooling. It is _crucial_ that no exception crates be dependencies
-// of the Rust runtime (std / test).
+/// These are exceptions to Rust's permissive licensing policy, and
+/// should be considered bugs. Exceptions are only allowed in Rust
+/// tooling. It is _crucial_ that no exception crates be dependencies
+/// of the Rust runtime (std / test).
 static EXCEPTIONS: &'static [&'static str] = &[
-    "mdbook", // MPL2, mdbook
-    "openssl", // BSD+advertising clause, cargo, mdbook
-    "pest", // MPL2, mdbook via handlebars
-    "thread-id", // Apache-2.0, mdbook
-    "toml-query", // MPL-2.0, mdbook
-    "is-match", // MPL-2.0, mdbook
-    "cssparser", // MPL-2.0, rustdoc
-    "smallvec", // MPL-2.0, rustdoc
+    "mdbook",             // MPL2, mdbook
+    "openssl",            // BSD+advertising clause, cargo, mdbook
+    "pest",               // MPL2, mdbook via handlebars
+    "thread-id",          // Apache-2.0, mdbook
+    "toml-query",         // MPL-2.0, mdbook
+    "is-match",           // MPL-2.0, mdbook
+    "cssparser",          // MPL-2.0, rustdoc
+    "smallvec",           // MPL-2.0, rustdoc
     "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
-    "fuchsia-zircon", // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
-    "cssparser-macros", // MPL-2.0, rustdoc
-    "selectors", // MPL-2.0, rustdoc
-    "clippy_lints", // MPL-2.0 rls
+    "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
+    "cssparser-macros",   // MPL-2.0, rustdoc
+    "selectors",          // MPL-2.0, rustdoc
+    "clippy_lints",       // MPL-2.0 rls
 ];
 
+/// Which crates to check against the whitelist?
+static WHITELIST_CRATES: &'static [CrateVersion] = &[
+    CrateVersion("rustc", "0.0.0"),
+    CrateVersion("rustc_trans", "0.0.0"),
+];
+
+/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
+static WHITELIST: &'static [Crate] = &[
+    Crate("ar"),
+    Crate("backtrace"),
+    Crate("backtrace-sys"),
+    Crate("bitflags"),
+    Crate("byteorder"),
+    Crate("cc"),
+    Crate("cfg-if"),
+    Crate("cmake"),
+    Crate("ena"),
+    Crate("filetime"),
+    Crate("flate2"),
+    Crate("fuchsia-zircon"),
+    Crate("fuchsia-zircon-sys"),
+    Crate("jobserver"),
+    Crate("kernel32-sys"),
+    Crate("lazy_static"),
+    Crate("libc"),
+    Crate("log"),
+    Crate("log_settings"),
+    Crate("miniz-sys"),
+    Crate("num_cpus"),
+    Crate("owning_ref"),
+    Crate("parking_lot"),
+    Crate("parking_lot_core"),
+    Crate("rand"),
+    Crate("redox_syscall"),
+    Crate("rustc-demangle"),
+    Crate("smallvec"),
+    Crate("stable_deref_trait"),
+    Crate("tempdir"),
+    Crate("unicode-width"),
+    Crate("winapi"),
+    Crate("winapi-build"),
+    Crate("winapi-i686-pc-windows-gnu"),
+    Crate("winapi-x86_64-pc-windows-gnu"),
+];
+
+// Some types for Serde to deserialize the output of `cargo metadata` to...
+
+#[derive(Deserialize)]
+struct Output {
+    resolve: Resolve,
+}
+
+#[derive(Deserialize)]
+struct Resolve {
+    nodes: Vec<ResolveNode>,
+}
+
+#[derive(Deserialize)]
+struct ResolveNode {
+    id: String,
+    dependencies: Vec<String>,
+}
+
+/// A unique identifier for a crate
+#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
+struct Crate<'a>(&'a str); // (name,)
+
+#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
+struct CrateVersion<'a>(&'a str, &'a str); // (name, version)
+
+impl<'a> Crate<'a> {
+    pub fn id_str(&self) -> String {
+        format!("{} ", self.0)
+    }
+}
+
+impl<'a> CrateVersion<'a> {
+    /// Returns the struct and whether or not the dep is in-tree
+    pub fn from_str(s: &'a str) -> (Self, bool) {
+        let mut parts = s.split(" ");
+        let name = parts.next().unwrap();
+        let version = parts.next().unwrap();
+        let path = parts.next().unwrap();
+
+        let is_path_dep = path.starts_with("(path+");
+
+        (CrateVersion(name, version), is_path_dep)
+    }
+
+    pub fn id_str(&self) -> String {
+        format!("{} {}", self.0, self.1)
+    }
+}
+
+impl<'a> From<CrateVersion<'a>> for Crate<'a> {
+    fn from(cv: CrateVersion<'a>) -> Crate<'a> {
+        Crate(cv.0)
+    }
+}
+
+/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
+///
+/// Specifically, this checks that the license is correct.
 pub fn check(path: &Path, bad: &mut bool) {
+    // Check licences
     let path = path.join("vendor");
     assert!(path.exists(), "vendor directory missing");
     let mut saw_dir = false;
-    'next_path: for dir in t!(path.read_dir()) {
+    for dir in t!(path.read_dir()) {
         saw_dir = true;
         let dir = t!(dir);
 
         // skip our exceptions
-        for exception in EXCEPTIONS {
-            if dir.path()
+        if EXCEPTIONS.iter().any(|exception| {
+            dir.path()
                 .to_str()
                 .unwrap()
-                .contains(&format!("src/vendor/{}", exception)) {
-                continue 'next_path;
-            }
+                .contains(&format!("src/vendor/{}", exception))
+        }) {
+            continue;
         }
 
         let toml = dir.path().join("Cargo.toml");
-        if !check_license(&toml) {
-            *bad = true;
-        }
+        *bad = *bad || !check_license(&toml);
     }
     assert!(saw_dir, "no vendored source");
 }
 
+/// Checks the dependency of WHITELIST_CRATES at the given path. Changes `bad` to `true` if a check
+/// failed.
+///
+/// Specifically, this checks that the dependencies are on the WHITELIST.
+pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
+    // Get dependencies from cargo metadata
+    let resolve = get_deps(path, cargo);
+
+    // Get the whitelist into a convenient form
+    let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();
+
+    // Check dependencies
+    let mut visited = BTreeSet::new();
+    let mut unapproved = BTreeSet::new();
+    for &krate in WHITELIST_CRATES.iter() {
+        let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false);
+        unapproved.append(&mut bad);
+    }
+
+    if unapproved.len() > 0 {
+        println!("Dependencies not on the whitelist:");
+        for dep in unapproved {
+            println!("* {}", dep.id_str());
+        }
+        *bad = true;
+    }
+}
+
 fn check_license(path: &Path) -> bool {
     if !path.exists() {
         panic!("{} does not exist", path.display());
@@ -102,9 +236,71 @@ fn extract_license(line: &str) -> String {
     let first_quote = line.find('"');
     let last_quote = line.rfind('"');
     if let (Some(f), Some(l)) = (first_quote, last_quote) {
-        let license = &line[f + 1 .. l];
+        let license = &line[f + 1..l];
         license.into()
     } else {
         "bad-license-parse".into()
     }
 }
+
+/// Get the dependencies of the crate at the given path using `cargo metadata`.
+fn get_deps(path: &Path, cargo: &Path) -> Resolve {
+    // Run `cargo metadata` to get the set of dependencies
+    let output = Command::new(cargo)
+        .arg("metadata")
+        .arg("--format-version")
+        .arg("1")
+        .arg("--manifest-path")
+        .arg(path.join("Cargo.toml"))
+        .output()
+        .expect("Unable to run `cargo metadata`")
+        .stdout;
+    let output = String::from_utf8_lossy(&output);
+    let output: Output = serde_json::from_str(&output).unwrap();
+
+    output.resolve
+}
+
+/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
+/// the whitelist. Returns a list of illegal dependencies.
+fn check_crate_whitelist<'a, 'b>(
+    whitelist: &'a HashSet<Crate>,
+    resolve: &'a Resolve,
+    visited: &'b mut BTreeSet<CrateVersion<'a>>,
+    krate: CrateVersion<'a>,
+    must_be_on_whitelist: bool,
+) -> BTreeSet<Crate<'a>> {
+    // Will contain bad deps
+    let mut unapproved = BTreeSet::new();
+
+    // Check if we have already visited this crate
+    if visited.contains(&krate) {
+        return unapproved;
+    }
+
+    visited.insert(krate);
+
+    // If this path is in-tree, we don't require it to be on the whitelist
+    if must_be_on_whitelist {
+        // If this dependency is not on the WHITELIST, add to bad set
+        if !whitelist.contains(&krate.into()) {
+            unapproved.insert(krate.into());
+        }
+    }
+
+    // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!)
+    let to_check = resolve
+        .nodes
+        .iter()
+        .find(|n| n.id.starts_with(&krate.id_str()))
+        .expect("crate does not exist");
+
+    for dep in to_check.dependencies.iter() {
+        let (krate, is_path_dep) = CrateVersion::from_str(dep);
+
+        let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
+        unapproved.append(&mut bad);
+    }
+
+    unapproved
+}

src/tools/tidy/src/lib.rs

@@ -15,6 +15,11 @@
 
 #![deny(warnings)]
 
+extern crate serde;
+extern crate serde_json;
+#[macro_use]
+extern crate serde_derive;
+
 use std::fs;
 
 use std::path::Path;

src/tools/tidy/src/main.rs

@@ -24,9 +24,12 @@
 use std::env;
 
 fn main() {
-    let path = env::args_os().skip(1).next().expect("need an argument");
+    let path = env::args_os().skip(1).next().expect("need path to src");
     let path = PathBuf::from(path);
 
+    let cargo = env::args_os().skip(2).next().expect("need path to cargo");
+    let cargo = PathBuf::from(cargo);
+
     let args: Vec<String> = env::args().skip(1).collect();
 
     let mut bad = false;
@@ -41,6 +44,7 @@ fn main() {
     if !args.iter().any(|s| *s == "--no-vendor") {
         deps::check(&path, &mut bad);
     }
+    deps::check_whitelist(&path, &cargo, &mut bad);
 
     if bad {
         eprintln!("some tidy checks failed");