Reuse existing download in y.sh prepare if fresh

Author: bjorn3

build_system/abi_cafe.rs

@@ -4,8 +4,13 @@ use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, CargoProject, Compiler};
 use super::{CodegenBackend, SysrootKind};
 
-static ABI_CAFE_REPO: GitRepo =
-    GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
+static ABI_CAFE_REPO: GitRepo = GitRepo::github(
+    "Gankra",
+    "abi-cafe",
+    "4c6dc8c9c687e2b3a760ff2176ce236872b37212",
+    "588df6d66abbe105",
+    "abi-cafe",
+);
 
 static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target");
 
@@ -18,6 +23,7 @@ pub(crate) fn run(
     bootstrap_host_compiler: &Compiler,
 ) {
     ABI_CAFE_REPO.fetch(dirs);
+    ABI_CAFE_REPO.patch(dirs);
 
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(

build_system/bench.rs

@@ -10,6 +10,7 @@ static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
     "ebobby",
     "simple-raytracer",
     "804a7a21b9e673a482797aa289a18ed480e4d813",
+    "ad6f59a2331a3f56",
     "<none>",
 );
 
@@ -24,9 +25,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
         std::process::exit(1);
     }
 
-    if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() {
-        SIMPLE_RAYTRACER_REPO.fetch(dirs);
-    }
+    SIMPLE_RAYTRACER_REPO.fetch(dirs);
+    SIMPLE_RAYTRACER_REPO.patch(dirs);
 
     let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
 

build_system/prepare.rs

@@ -10,14 +10,18 @@ use super::tests::LIBCORE_TESTS_SRC;
 use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
 
 pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) {
-    RelPath::DOWNLOAD.ensure_fresh(dirs);
+    RelPath::DOWNLOAD.ensure_exists(dirs);
+    super::tests::RAND_REPO.fetch(dirs);
+    super::tests::REGEX_REPO.fetch(dirs);
+    super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
 
+    // FIXME do this on the fly?
     prepare_stdlib(dirs, rustc);
     prepare_coretests(dirs, rustc);
 
-    super::tests::RAND_REPO.fetch(dirs);
-    super::tests::REGEX_REPO.fetch(dirs);
-    super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
+    super::tests::RAND_REPO.patch(dirs);
+    super::tests::REGEX_REPO.patch(dirs);
+    super::tests::PORTABLE_SIMD_REPO.patch(dirs);
 }
 
 fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {
@@ -61,21 +65,57 @@ fn prepare_coretests(dirs: &Dirs, rustc: &Path) {
 pub(crate) struct GitRepo {
     url: GitRepoUrl,
     rev: &'static str,
+    content_hash: &'static str,
     patch_name: &'static str,
 }
 
 enum GitRepoUrl {
     Github { user: &'static str, repo: &'static str },
 }
 
+// Note: This uses a hasher which is not cryptographically secure. This is fine as the hash is meant
+// to protect against accidental modification and outdated downloads, not against manipulation.
+fn hash_file(file: &std::path::Path) -> u64 {
+    let contents = std::fs::read(file).unwrap();
+    #[allow(deprecated)]
+    let mut hasher = std::hash::SipHasher::new();
+    std::hash::Hash::hash(&contents, &mut hasher);
+    std::hash::Hasher::finish(&hasher)
+}
+
+fn hash_dir(dir: &std::path::Path) -> u64 {
+    let mut sub_hashes = std::collections::BTreeMap::new();
+    for entry in std::fs::read_dir(dir).unwrap() {
+        let entry = entry.unwrap();
+        if entry.file_type().unwrap().is_dir() {
+            sub_hashes
+                .insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path()));
+        } else {
+            sub_hashes
+                .insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path()));
+        }
+    }
+    #[allow(deprecated)]
+    let mut hasher = std::hash::SipHasher::new();
+    std::hash::Hash::hash(&sub_hashes, &mut hasher);
+    std::hash::Hasher::finish(&hasher)
+}
+
 impl GitRepo {
     pub(crate) const fn github(
         user: &'static str,
         repo: &'static str,
         rev: &'static str,
+        content_hash: &'static str,
         patch_name: &'static str,
     ) -> GitRepo {
-        GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name }
+        GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name }
+    }
+
+    fn download_dir(&self, dirs: &Dirs) -> PathBuf {
+        match self.url {
+            GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs),
+        }
     }
 
     pub(crate) const fn source_dir(&self) -> RelPath {
@@ -85,15 +125,42 @@ impl GitRepo {
     }
 
     pub(crate) fn fetch(&self, dirs: &Dirs) {
-        let download_dir = match self.url {
-            GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs),
-        };
-        let source_dir = self.source_dir();
+        let download_dir = self.download_dir(dirs);
+
+        if download_dir.exists() {
+            let actual_hash = format!("{:016x}", hash_dir(&download_dir));
+            if actual_hash == self.content_hash {
+                println!("[FRESH] {}", download_dir.display());
+                return;
+            } else {
+                println!(
+                    "Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.",
+                    download_dir = download_dir.display(),
+                    content_hash = self.content_hash,
+                );
+            }
+        }
+
         match self.url {
             GitRepoUrl::Github { user, repo } => {
                 clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev);
             }
         }
+
+        let actual_hash = format!("{:016x}", hash_dir(&download_dir));
+        if actual_hash != self.content_hash {
+            println!(
+                "Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}",
+                download_dir = download_dir.display(),
+                content_hash = self.content_hash,
+            );
+            std::process::exit(1);
+        }
+    }
+
+    pub(crate) fn patch(&self, dirs: &Dirs) {
+        let download_dir = self.download_dir(dirs);
+        let source_dir = self.source_dir();
         source_dir.ensure_fresh(dirs);
         copy_dir_recursively(&download_dir, &source_dir.to_path(dirs));
         apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs));

build_system/tests.rs

@@ -94,20 +94,31 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
 
 // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is
 // fixed.
-pub(crate) static RAND_REPO: GitRepo =
-    GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand");
+pub(crate) static RAND_REPO: GitRepo = GitRepo::github(
+    "rust-random",
+    "rand",
+    "50b9a447410860af8d6db9a208c3576886955874",
+    "98b2276210b30e43",
+    "rand",
+);
 
 pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target");
 
-pub(crate) static REGEX_REPO: GitRepo =
-    GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex");
+pub(crate) static REGEX_REPO: GitRepo = GitRepo::github(
+    "rust-lang",
+    "regex",
+    "32fed9429eafba0ae92a64b01796a0c5a75b88c8",
+    "d6af6507d565aa66",
+    "regex",
+);
 
 pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex_target");
 
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
     "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b",
+    "1ba291009510070b",
     "portable-simd",
 );