Auto merge of #116635 - onur-ozkan:improve-file-read, r=Mark-Simulacrum

optimize file read in `Config::verify`

`Config::verify` refactored to improve the efficiency and memory usage of file hashing.

src/bootstrap/config/tests.rs

@@ -3,7 +3,12 @@ use crate::config::TomlConfig;
 use super::{Config, Flags};
 use clap::CommandFactory;
 use serde::Deserialize;
-use std::{env, path::Path};
+use std::{
+    env,
+    fs::{remove_file, File},
+    io::Write,
+    path::Path,
+};
 
 fn parse(config: &str) -> Config {
     Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], |&_| {
@@ -196,3 +201,19 @@ fn rust_optimize() {
 fn invalid_rust_optimize() {
     parse("rust.optimize = \"a\"");
 }
+
+#[test]
+fn verify_file_integrity() {
+    let config = parse("");
+
+    let tempfile = config.tempdir().join(".tmp-test-file");
+    File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap();
+    assert!(tempfile.exists());
+
+    assert!(
+        config
+            .verify(&tempfile, "7e255dd9542648a8779268a0f268b891a198e9828e860ed23f826440e786eae5")
+    );
+
+    remove_file(tempfile).unwrap();
+}

src/bootstrap/download.rs

@@ -320,25 +320,43 @@ impl Config {
     }
 
     /// Returns whether the SHA256 checksum of `path` matches `expected`.
-    fn verify(&self, path: &Path, expected: &str) -> bool {
+    pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
         use sha2::Digest;
 
         self.verbose(&format!("verifying {}", path.display()));
+
+        if self.dry_run() {
+            return false;
+        }
+
         let mut hasher = sha2::Sha256::new();
-        // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
+
-        // Consider using streaming IO instead?
+        let file = t!(File::open(path));
-        let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) };
+        let mut reader = BufReader::new(file);
-        hasher.update(&contents);
+
-        let found = hex::encode(hasher.finalize().as_slice());
+        loop {
-        let verified = found == expected;
+            let buffer = t!(reader.fill_buf());
-        if !verified && !self.dry_run() {
+            let l = buffer.len();
+            // break if EOF
+            if l == 0 {
+                break;
+            }
+            hasher.update(buffer);
+            reader.consume(l);
+        }
+
+        let checksum = hex::encode(hasher.finalize().as_slice());
+        let verified = checksum == expected;
+
+        if !verified {
             println!(
                 "invalid checksum: \n\
-                found:    {found}\n\
+                found:    {checksum}\n\
                 expected: {expected}",
             );
         }
-        return verified;
+
+        verified
     }
 }