tools: use SHA256 in ci_download_cockroachdb (#5017)

Depends on #4961.

Currently, the `tools/ci_download_cockroachdb` script uses MD5 as the
checksum for the Cockroachdb tarball. This is unfortunate for two
reasons:

1. Upstream Cockroachdb (and the corresponding Illumos build) publish
SHA256 digests for these tarballs, rather than MD5s (see
https://www.cockroachlabs.com/docs/releases/ and
https://illumos.org/downloads/). Using SHA256 rather than MD5 digests
should make updating to a new version easier.

2. Nix requires SHA256 checksums for files downloaded as build inputs.
Currently, the Nix flake can use the SHA256 checksums for Maghemite and
Dendrite from `tools/maghemite_mgd_checksums` and
`tools/dendrite_stub_checksums`, meaning that updating these versions
does not require manually changing the Nix flake. However, because we
use MD5 rather than SHA256 checksums for Cockroachdb, updating the
Cockroachdb version requires manually changing the version in the Nix
flake, which is a shame, especially if someone unfamiliar with Nix has
to do it...

This commit changes `tools/cockroachdb_checksums` and the corresponding
`tools/ci_download_cockroachdb` script to use SHA256 rather than MD5.
I've changed the Nix flake to read the hash from this file rather than
hard-coding it, so now, the cockroachdb version can be updated without
touching the flake.

flake.nix

@@ -98,6 +98,8 @@
           version = mgVersion;
         };
 
+      # given a list of strings of the form `PREFIX="SHA256"`, finds the string
+      # starting with the provided `name` and returns the hash for that prefix.
       findSha = with pkgs.lib;
         shas: (name:
           let
@@ -119,9 +121,7 @@
               file = builtins.readFile
                 ./tools/dendrite_stub_checksums;
             in
-            strings.splitString
-              "\n"
-              file;
+            strings.splitString "\n" file;
           findStubSha = name: findSha stubShas "CIDL_SHA256_${name}";
           fetchLinuxBin = file:
             downloadBuildomat {
@@ -288,10 +288,16 @@
           name = "cockroachdb";
           binName = "cockroach";
           version = readVersionFile "${name}_version";
+          sha256 =
+            let
+              shaFile = builtins.readFile  ./tools/${name}_checksums;
+              shas = lib.strings.splitString "\n" shaFile;
+            in
+            findSha shas "CIDL_SHA256_LINUX";
           src = builtins.fetchurl
             {
+              inherit sha256;
               url = "https://binaries.cockroachdb.com/${binName}-v${version}.linux-amd64.tgz";
-              sha256 = "1aglbwh27275bicyvij11s3as4zypqwc26p9gyh5zr3y1s123hr4";
             };
         in
         stdenv.mkDerivation

tools/ci_download_cockroachdb

@@ -13,7 +13,7 @@ set -o errexit
 SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 ARG0="$(basename "${BASH_SOURCE[0]}")"
 
-# If you change this, you must also update the md5sums below
+# If you change this, you must also update the sha256sums below
 CIDL_VERSION="$(cat "$SOURCE_DIR/cockroachdb_version")"
 source "$SOURCE_DIR/cockroachdb_checksums"
 
@@ -49,6 +49,7 @@ function main
 	# Configure this program
 	configure_os "$CIDL_OS"
 	CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME"
+	CIDL_SHA256FUNC="do_sha256sum"
 
 	# Download the file.
 	echo "URL: $CIDL_URL"
@@ -60,9 +61,9 @@ function main
 	local DO_DOWNLOAD="true"
 	if [[ -f "$TARBALL_FILE" ]]; then
 		# If the file exists with a valid checksum, we can skip downloading.
-		calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
-				fail "failed to calculate md5sum"
-		if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then
+		calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate sha256sum"
+		if [[ "$calculated_sha256" == "$CIDL_SHA256" ]]; then
 			DO_DOWNLOAD="false"
 		fi
 	fi
@@ -72,12 +73,12 @@ function main
 		do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \
 				fail "failed to download file"
 
-		# Verify the md5sum.
-		calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \
-				fail "failed to calculate md5sum"
-		if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then
-			fail "md5sum mismatch \
-					(expected $CIDL_MD5, found $calculated_md5)"
+		# Verify the sha256sum.
+		calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate sha256sum"
+		if [[ "$calculated_sha256" != "$CIDL_SHA256" ]]; then
+			fail "sha256sum mismatch \
+					(expected $CIDL_SHA256, found $calculated_sha256)"
 		fi
 	fi
 
@@ -105,24 +106,21 @@ function configure_os
 		darwin*)
 			CIDL_BUILD="darwin-10.9-amd64"
 			CIDL_SUFFIX="tgz"
-			CIDL_MD5="$CIDL_MD5_DARWIN"
-			CIDL_MD5FUNC="do_md5"
+			CIDL_SHA256="$CIDL_SHA256_DARWIN"
 			CIDL_URL_BASE="$CIDL_URL_COCKROACH"
 			CIDL_ASSEMBLE="do_assemble_official"
 			;;
 		linux-gnu*)
 			CIDL_BUILD="linux-amd64"
 			CIDL_SUFFIX="tgz"
-			CIDL_MD5="$CIDL_MD5_LINUX"
-			CIDL_MD5FUNC="do_md5sum"
+			CIDL_SHA256="$CIDL_SHA256_LINUX"
 			CIDL_URL_BASE="$CIDL_URL_COCKROACH"
 			CIDL_ASSEMBLE="do_assemble_official"
 			;;
 		solaris*)
 			CIDL_BUILD="illumos"
 			CIDL_SUFFIX="tar.gz"
-			CIDL_MD5="$CIDL_MD5_ILLUMOS"
-			CIDL_MD5FUNC="do_md5sum"
+			CIDL_SHA256="$CIDL_SHA256_ILLUMOS"
 			CIDL_URL_BASE="$CIDL_URL_ILLUMOS"
 			CIDL_ASSEMBLE="do_assemble_illumos"
 			;;
@@ -143,14 +141,9 @@ function do_download_curl
 	curl --silent --show-error --fail --location --output "$2" "$1"
 }
 
-function do_md5
+function do_sha256sum
 {
-	md5 < "$1"
-}
-
-function do_md5sum
-{
-	md5sum < "$1" | awk '{print $1}'
+	sha256sum < "$1" | awk '{print $1}'
 }
 
 function do_untar

tools/cockroachdb_checksums

@@ -1,3 +1,3 @@
-CIDL_MD5_DARWIN="2db972c254b4e3b599e12110520178b5"
-CIDL_MD5_LINUX="8c3170883e0a0be1a34b44090c067a8c"
-CIDL_MD5_ILLUMOS="d8999aff364e5d70f226e139fda724a3"
+CIDL_SHA256_DARWIN="1ca69e0911af11a73305c3c6f4650b912d70754900b5bf7b80a1d361efe36561"
+CIDL_SHA256_LINUX="24c321820e7ee45fa07fe91ac138befe13ad860e41c6ed595ce58823205ff4a9"
+CIDL_SHA256_ILLUMOS="f151714ba3a6e02caaaa59727482c36085e60d6bd2fa963938e9a3d8c8a77088"