Auto merge of #12939 - Alexendoo:lintcheck-popular-crates, r=xFrednet

Merge lintcheck popular-crates bin as a subcommand

Also rewrites it to use `ureq` to drop some heavy dependencies as `crates_io_api` brings in `reqwest`

r? `@xFrednet`

changelog: none

lintcheck/Cargo.toml

@@ -11,30 +11,19 @@ publish = false
 default-run = "lintcheck"
 
 [dependencies]
-anyhow = "1.0.69"
 cargo_metadata = "0.15.3"
 clap = { version = "4.4", features = ["derive", "env"] }
-crates_io_api = "0.8.1"
 crossbeam-channel = "0.5.6"
 diff = "0.1.13"
 flate2 = "1.0"
-indicatif = "0.17.3"
 rayon = "1.5.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.85"
 strip-ansi-escapes = "0.1.1"
 tar = "0.4"
 toml = "0.7.3"
-ureq = "2.2"
+ureq = { version = "2.2", features = ["json"] }
 walkdir = "2.3"
 
 [features]
 deny-warnings = []
-
-[[bin]]
-name = "lintcheck"
-path = "src/main.rs"
-
-[[bin]]
-name = "popular-crates"
-path = "src/popular-crates.rs"

lintcheck/README.md

@@ -26,11 +26,11 @@ the repo root.
 The results will then be saved to `lintcheck-logs/custom_logs.toml`.
 
 The `custom.toml` file may be built using <https://crates.io> recently most
-downloaded crates by using the `popular-crates` binary from the `lintcheck`
-directory. For example, to retrieve the 100 recently most downloaded crates:
+downloaded crates by using `cargo lintcheck popular`. For example, to retrieve
+the 200 recently most downloaded crates:
 
 ```
-cargo run --release --bin popular-crates -- -n 100 custom.toml
+cargo lintcheck popular -n 200 custom.toml
 ```
 
 

lintcheck/src/config.rs

@@ -48,7 +48,16 @@ pub(crate) struct LintcheckConfig {
 
 #[derive(Subcommand, Clone, Debug)]
 pub(crate) enum Commands {
+    /// Display a markdown diff between two lintcheck log files in JSON format
     Diff { old: PathBuf, new: PathBuf },
+    /// Create a lintcheck crates TOML file containing the top N popular crates
+    Popular {
+        /// Output TOML file name
+        output: PathBuf,
+        /// Number of crate names to download
+        #[clap(short, long, default_value_t = 100)]
+        number: usize,
+    },
 }
 
 #[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq)]

lintcheck/src/main.rs

@@ -17,6 +17,7 @@
 mod config;
 mod driver;
 mod json;
+mod popular_crates;
 mod recursive;
 
 use crate::config::{Commands, LintcheckConfig, OutputFormat};
@@ -43,21 +44,21 @@
 const LINTCHECK_SOURCES: &str = "target/lintcheck/sources";
 
 /// List of sources to check, loaded from a .toml file
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Deserialize)]
 struct SourceList {
     crates: HashMap<String, TomlCrate>,
     #[serde(default)]
     recursive: RecursiveOptions,
 }
 
-#[derive(Debug, Serialize, Deserialize, Default)]
+#[derive(Debug, Deserialize, Default)]
 struct RecursiveOptions {
     ignore: HashSet<String>,
 }
 
 /// A crate source stored inside the .toml
 /// will be translated into on one of the `CrateSource` variants
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Deserialize)]
 struct TomlCrate {
     name: String,
     versions: Option<Vec<String>>,
@@ -69,7 +70,7 @@ struct TomlCrate {
 
 /// Represents an archive we download from crates.io, or a git repo, or a local repo/folder
 /// Once processed (downloaded/extracted/cloned/copied...), this will be translated into a `Crate`
-#[derive(Debug, Serialize, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)]
 enum CrateSource {
     CratesIo {
         name: String,
@@ -609,7 +610,6 @@ fn gather_stats(warnings: &[ClippyWarning]) -> (String, HashMap<&String, usize>)
     (stats_string, counter)
 }
 
-#[allow(clippy::too_many_lines)]
 fn main() {
     // We're being executed as a `RUSTC_WRAPPER` as part of `--recursive`
     if let Ok(addr) = env::var("LINTCHECK_SERVER") {
@@ -624,11 +624,15 @@ fn main() {
 
     let config = LintcheckConfig::new();
 
-    if let Some(Commands::Diff { old, new }) = config.subcommand {
-        json::diff(&old, &new);
-        return;
+    match config.subcommand {
+        Some(Commands::Diff { old, new }) => json::diff(&old, &new),
+        Some(Commands::Popular { output, number }) => popular_crates::fetch(output, number).unwrap(),
+        None => lintcheck(config),
     }
+}
 
+#[allow(clippy::too_many_lines)]
+fn lintcheck(config: LintcheckConfig) {
     println!("Compiling clippy...");
     build_clippy();
     println!("Done compiling");

lintcheck/src/popular-crates.rs

@@ -1,65 +0,0 @@
-#![deny(clippy::pedantic)]
-
-use clap::Parser;
-use crates_io_api::{CratesQueryBuilder, Sort, SyncClient};
-use indicatif::ProgressBar;
-use std::collections::HashSet;
-use std::fs::File;
-use std::io::{BufWriter, Write};
-use std::path::PathBuf;
-use std::time::Duration;
-
-#[derive(Parser)]
-struct Opts {
-    /// Output TOML file name
-    output: PathBuf,
-    /// Number of crate names to download
-    #[clap(short, long, default_value_t = 100)]
-    number: usize,
-    /// Do not output progress
-    #[clap(short, long)]
-    quiet: bool,
-}
-
-fn main() -> anyhow::Result<()> {
-    let opts = Opts::parse();
-    let mut output = BufWriter::new(File::create(opts.output)?);
-    output.write_all(b"[crates]\n")?;
-    let client = SyncClient::new(
-        "clippy/lintcheck (github.com/rust-lang/rust-clippy/)",
-        Duration::from_secs(1),
-    )?;
-    let mut seen_crates = HashSet::new();
-    let pb = if opts.quiet {
-        None
-    } else {
-        Some(ProgressBar::new(opts.number as u64))
-    };
-    let mut query = CratesQueryBuilder::new()
-        .sort(Sort::RecentDownloads)
-        .page_size(100)
-        .build();
-    while seen_crates.len() < opts.number {
-        let retrieved = client.crates(query.clone())?.crates;
-        if retrieved.is_empty() {
-            eprintln!("No more than {} crates available from API", seen_crates.len());
-            break;
-        }
-        for c in retrieved {
-            if seen_crates.insert(c.name.clone()) {
-                output.write_all(
-                    format!(
-                        "{} = {{ name = '{}', versions = ['{}'] }}\n",
-                        c.name, c.name, c.max_version
-                    )
-                    .as_bytes(),
-                )?;
-                if let Some(pb) = &pb {
-                    pb.inc(1);
-                }
-            }
-        }
-        query.set_page(query.page() + 1);
-    }
-    Ok(())
-}

lintcheck/src/popular_crates.rs

@@ -0,0 +1,52 @@
+use serde::Deserialize;
+use std::error::Error;
+use std::fmt::Write;
+use std::fs;
+use std::path::PathBuf;
+
+#[derive(Deserialize, Debug)]
+struct Page {
+    crates: Vec<Crate>,
+    meta: Meta,
+}
+
+#[derive(Deserialize, Debug)]
+struct Crate {
+    name: String,
+    max_version: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct Meta {
+    next_page: String,
+}
+
+pub(crate) fn fetch(output: PathBuf, number: usize) -> Result<(), Box<dyn Error>> {
+    let agent = ureq::builder()
+        .user_agent("clippy/lintcheck (github.com/rust-lang/rust-clippy/)")
+        .build();
+
+    let mut crates = Vec::with_capacity(number);
+    let mut query = "?sort=recent-downloads&per_page=100".to_string();
+    while crates.len() < number {
+        let page: Page = agent
+            .get(&format!("https://crates.io/api/v1/crates{query}"))
+            .call()?
+            .into_json()?;
+
+        query = page.meta.next_page;
+        crates.extend(page.crates);
+        crates.truncate(number);
+
+        let width = number.ilog10() as usize + 1;
+        println!("Fetched {:>width$}/{number} crates", crates.len());
+    }
+
+    let mut out = "[crates]\n".to_string();
+    for Crate { name, max_version } in crates {
+        writeln!(out, "{name} = {{ name = '{name}', versions = ['{max_version}'] }}").unwrap();
+    }
+    fs::write(output, out)?;
+
+    Ok(())
+}