From 75327f8587fad77a25cfe99e52376b6862d75b57 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:49:41 +0000 Subject: [PATCH] Reuse existing download in y.sh prepare if fresh --- build_system/abi_cafe.rs | 10 ++++- build_system/bench.rs | 6 +-- build_system/prepare.rs | 87 +++++++++++++++++++++++++++++++++++----- build_system/tests.rs | 19 +++++++-- 4 files changed, 103 insertions(+), 19 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index ba15db9f966..29c127bf50e 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -4,8 +4,13 @@ use super::utils::{spawn_and_wait, CargoProject, Compiler}; use super::{CodegenBackend, SysrootKind}; -static ABI_CAFE_REPO: GitRepo = - GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); +static ABI_CAFE_REPO: GitRepo = GitRepo::github( + "Gankra", + "abi-cafe", + "4c6dc8c9c687e2b3a760ff2176ce236872b37212", + "588df6d66abbe105", + "abi-cafe", +); static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); @@ -18,6 +23,7 @@ pub(crate) fn run( bootstrap_host_compiler: &Compiler, ) { ABI_CAFE_REPO.fetch(dirs); + ABI_CAFE_REPO.patch(dirs); eprintln!("Building sysroot for abi-cafe"); build_sysroot::build_sysroot( diff --git a/build_system/bench.rs b/build_system/bench.rs index d24803eb7c6..2bb11800034 100644 --- a/build_system/bench.rs +++ b/build_system/bench.rs @@ -10,6 +10,7 @@ "ebobby", "simple-raytracer", "804a7a21b9e673a482797aa289a18ed480e4d813", + "ad6f59a2331a3f56", "", ); @@ -24,9 +25,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { std::process::exit(1); } - if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() { - SIMPLE_RAYTRACER_REPO.fetch(dirs); - } + SIMPLE_RAYTRACER_REPO.fetch(dirs); + SIMPLE_RAYTRACER_REPO.patch(dirs); let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 7bb9eca2945..b76164a1d00 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -10,14 +10,18 @@ use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait}; pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { - RelPath::DOWNLOAD.ensure_fresh(dirs); - - prepare_stdlib(dirs, rustc); - prepare_coretests(dirs, rustc); - + RelPath::DOWNLOAD.ensure_exists(dirs); super::tests::RAND_REPO.fetch(dirs); super::tests::REGEX_REPO.fetch(dirs); super::tests::PORTABLE_SIMD_REPO.fetch(dirs); + + // FIXME do this on the fly? + prepare_stdlib(dirs, rustc); + prepare_coretests(dirs, rustc); + + super::tests::RAND_REPO.patch(dirs); + super::tests::REGEX_REPO.patch(dirs); + super::tests::PORTABLE_SIMD_REPO.patch(dirs); } fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { @@ -61,6 +65,7 @@ fn prepare_coretests(dirs: &Dirs, rustc: &Path) { pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, + content_hash: &'static str, patch_name: &'static str, } @@ -68,14 +73,49 @@ enum GitRepoUrl { Github { user: &'static str, repo: &'static str }, } +// Note: This uses a hasher which is not cryptographically secure. This is fine as the hash is meant +// to protect against accidental modification and outdated downloads, not against manipulation. +fn hash_file(file: &std::path::Path) -> u64 { + let contents = std::fs::read(file).unwrap(); + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + std::hash::Hash::hash(&contents, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + +fn hash_dir(dir: &std::path::Path) -> u64 { + let mut sub_hashes = std::collections::BTreeMap::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + sub_hashes + .insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path())); + } else { + sub_hashes + .insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path())); + } + } + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + std::hash::Hash::hash(&sub_hashes, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + impl GitRepo { pub(crate) const fn github( user: &'static str, repo: &'static str, rev: &'static str, + content_hash: &'static str, patch_name: &'static str, ) -> GitRepo { - GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name } + GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name } + } + + fn download_dir(&self, dirs: &Dirs) -> PathBuf { + match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), + } } pub(crate) const fn source_dir(&self) -> RelPath { @@ -85,15 +125,42 @@ pub(crate) const fn source_dir(&self) -> RelPath { } pub(crate) fn fetch(&self, dirs: &Dirs) { - let download_dir = match self.url { - GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), - }; - let source_dir = self.source_dir(); + let download_dir = self.download_dir(dirs); + + if download_dir.exists() { + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash == self.content_hash { + println!("[FRESH] {}", download_dir.display()); + return; + } else { + println!( + "Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + } + } + match self.url { GitRepoUrl::Github { user, repo } => { clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev); } } + + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash != self.content_hash { + println!( + "Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + std::process::exit(1); + } + } + + pub(crate) fn patch(&self, dirs: &Dirs) { + let download_dir = self.download_dir(dirs); + let source_dir = self.source_dir(); source_dir.ensure_fresh(dirs); copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); diff --git a/build_system/tests.rs b/build_system/tests.rs index f975e43b13d..e60f0a4c612 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -94,13 +94,23 @@ const fn jit_bin(config: &'static str, source: &'static str, args: &'static str) // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is // fixed. -pub(crate) static RAND_REPO: GitRepo = - GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand"); +pub(crate) static RAND_REPO: GitRepo = GitRepo::github( + "rust-random", + "rand", + "50b9a447410860af8d6db9a208c3576886955874", + "98b2276210b30e43", + "rand", +); pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target"); -pub(crate) static REGEX_REPO: GitRepo = - GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex"); +pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( + "rust-lang", + "regex", + "32fed9429eafba0ae92a64b01796a0c5a75b88c8", + "d6af6507d565aa66", + "regex", +); pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex_target"); @@ -108,6 +118,7 @@ const fn jit_bin(config: &'static str, source: &'static str, args: &'static str) "rust-lang", "portable-simd", "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", + "1ba291009510070b", "portable-simd", );