switch clickhouse downloads to sha256 checksums (#5959)

Also hooked up flake.nix to the checksums file now that it's sha256.
Checked the Nix flake with `nix flake check` and found that rust-overlay
was too out of date to have 1.78.0, so bumped that as well (`nix flake
lock --update-input rust-overlay`).

dev-tools/xtask/Cargo.toml

@@ -17,7 +17,6 @@ flate2.workspace = true
 futures.workspace = true
 fs-err.workspace = true
 macaddr.workspace = true
-md5 = "0.7.0"
 reqwest = { workspace = true, features = [ "default-tls" ] }
 serde.workspace = true
 sha2.workspace = true

dev-tools/xtask/src/download.rs

@@ -257,23 +257,6 @@ async fn streaming_download(url: &str, path: &Utf8Path) -> Result<()> {
     Ok(())
 }
 
-/// Returns the hex, lowercase md5 checksum of a file at `path`.
-async fn md5_checksum(path: &Utf8Path) -> Result<String> {
-    let mut buf = vec![0u8; 65536];
-    let mut file = tokio::fs::File::open(path).await?;
-    let mut ctx = md5::Context::new();
-    loop {
-        let n = file.read(&mut buf).await?;
-        if n == 0 {
-            break;
-        }
-        ctx.write_all(&buf[0..n])?;
-    }
-
-    let digest = ctx.compute();
-    Ok(format!("{digest:x}"))
-}
-
 /// Returns the hex, lowercase sha2 checksum of a file at `path`.
 async fn sha2_checksum(path: &Utf8Path) -> Result<String> {
     let mut buf = vec![0u8; 65536];
@@ -382,14 +365,12 @@ async fn set_permissions(path: &Utf8Path, mode: u32) -> Result<()> {
 }
 
 enum ChecksumAlgorithm {
-    Md5,
     Sha2,
 }
 
 impl ChecksumAlgorithm {
     async fn checksum(&self, path: &Utf8Path) -> Result<String> {
         match self {
-            ChecksumAlgorithm::Md5 => md5_checksum(path).await,
             ChecksumAlgorithm::Sha2 => sha2_checksum(path).await,
         }
     }
@@ -459,7 +440,7 @@ impl<'a> Downloader<'a> {
 
         let checksums_path = self.versions_dir.join("clickhouse_checksums");
         let [checksum] = get_values_from_file(
-            [&format!("CIDL_MD5_{}", os.env_name())],
+            [&format!("CIDL_SHA256_{}", os.env_name())],
             &checksums_path,
         )
         .await?;
@@ -491,7 +472,7 @@ impl<'a> Downloader<'a> {
             &self.log,
             &tarball_path,
             &tarball_url,
-            ChecksumAlgorithm::Md5,
+            ChecksumAlgorithm::Sha2,
             &checksum,
         )
         .await?;

flake.nix

@@ -255,12 +255,12 @@
         let
           name = "clickhouse";
           version = readVersionFile "${name}_version";
-          # N.B. that unlike maghemite and dendrite, the Clickhouse hashes
-          # in `tools/clickhouse_checksums` are MD5 rather than SHA256, so we
-          # can't give Nix those hashes and must instead determine it ourselves.
-          # this means that we will have to update this SHA if the clickhouse
-          # version changes.
-          sha256 = "0wx8w9sdms5hsc9f835ivsissf15wjzdb9cvxr65xdi384i9pkzx";
+          sha256 =
+            let
+              shaFile = builtins.readFile  ./tools/${name}_checksums;
+              shas = lib.strings.splitString "\n" shaFile;
+            in
+            findSha shas "CIDL_SHA256_LINUX";
           src = builtins.fetchurl
             {
               inherit sha256;

tools/clickhouse_checksums

@@ -1,3 +1,3 @@
-CIDL_MD5_DARWIN="3e20c3284b7e6b0cfcfedf622ecf547a"
-CIDL_MD5_LINUX="f6c30a25a86deac3bad6c50dcf758fd5"
-CIDL_MD5_ILLUMOS="409222de8ecb59e5dd97dcc942ccdffe"
+CIDL_SHA256_DARWIN="275843f5942bf84a27cfededa2314d70d4a3300d7762045e1f1b5cd93e97c6a1"
+CIDL_SHA256_LINUX="fdcf9b224123b65e4cee9ba5d5bee42538ada3deb10ce412d3b0e8da74e2a873"
+CIDL_SHA256_ILLUMOS="29ad158e985b088f6b8987b33dac96dcbaed9bdffc95ae0e880273cc125320b3"