From 00d802453d79053318e41a77c8056b7c014f18c2 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 7 Feb 2024 16:03:55 -0800 Subject: [PATCH] tools: use SHA256 in `ci_download_cockroachdb` (#5017) Depends on #4961. Currently, the `tools/ci_download_cockroachdb` script uses MD5 as the checksum for the Cockroachdb tarball. This is unfortunate for two reasons: 1. Upstream Cockroachdb (and the corresponding Illumos build) publish SHA256 digests for these tarballs, rather than MD5s (see https://www.cockroachlabs.com/docs/releases/ and https://illumos.org/downloads/). Using SHA256 rather than MD5 digests should make updating to a new version easier. 2. Nix requires SHA256 checksums for files downloaded as build inputs. Currently, the Nix flake can use the SHA256 checksums for Maghemite and Dendrite from `tools/maghemite_mgd_checksums` and `tools/dendrite_stub_checksums`, meaning that updating these versions does not require manually changing the Nix flake. However, because we use MD5 rather than SHA256 checksums for Cockroachdb, updating the Cockroachdb version requires manually changing the version in the Nix flake, which is a shame, especially if someone unfamiliar with Nix has to do it... This commit changes `tools/cockroachdb_checksums` and the corresponding `tools/ci_download_cockroachdb` script to use SHA256 rather than MD5. I've changed the Nix flake to read the hash from this file rather than hard-coding it, so now, the cockroachdb version can be updated without touching the flake. --- flake.nix | 14 +++++++++---- tools/ci_download_cockroachdb | 39 ++++++++++++++--------------------- tools/cockroachdb_checksums | 6 +++--- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/flake.nix b/flake.nix index 1f9a992274..408dff5706 100644 --- a/flake.nix +++ b/flake.nix @@ -98,6 +98,8 @@ version = mgVersion; }; + # given a list of strings of the form `PREFIX="SHA256"`, finds the string + # starting with the provided `name` and returns the hash for that prefix. findSha = with pkgs.lib; shas: (name: let @@ -119,9 +121,7 @@ file = builtins.readFile ./tools/dendrite_stub_checksums; in - strings.splitString - "\n" - file; + strings.splitString "\n" file; findStubSha = name: findSha stubShas "CIDL_SHA256_${name}"; fetchLinuxBin = file: downloadBuildomat { @@ -288,10 +288,16 @@ name = "cockroachdb"; binName = "cockroach"; version = readVersionFile "${name}_version"; + sha256 = + let + shaFile = builtins.readFile ./tools/${name}_checksums; + shas = lib.strings.splitString "\n" shaFile; + in + findSha shas "CIDL_SHA256_LINUX"; src = builtins.fetchurl { + inherit sha256; url = "https://binaries.cockroachdb.com/${binName}-v${version}.linux-amd64.tgz"; - sha256 = "1aglbwh27275bicyvij11s3as4zypqwc26p9gyh5zr3y1s123hr4"; }; in stdenv.mkDerivation diff --git a/tools/ci_download_cockroachdb b/tools/ci_download_cockroachdb index ca484c000f..5755e7e665 100755 --- a/tools/ci_download_cockroachdb +++ b/tools/ci_download_cockroachdb @@ -13,7 +13,7 @@ set -o errexit SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" ARG0="$(basename "${BASH_SOURCE[0]}")" -# If you change this, you must also update the md5sums below +# If you change this, you must also update the sha256sums below CIDL_VERSION="$(cat "$SOURCE_DIR/cockroachdb_version")" source "$SOURCE_DIR/cockroachdb_checksums" @@ -49,6 +49,7 @@ function main # Configure this program configure_os "$CIDL_OS" CIDL_URL="$CIDL_URL_BASE/$TARBALL_FILENAME" + CIDL_SHA256FUNC="do_sha256sum" # Download the file. echo "URL: $CIDL_URL" @@ -60,9 +61,9 @@ function main local DO_DOWNLOAD="true" if [[ -f "$TARBALL_FILE" ]]; then # If the file exists with a valid checksum, we can skip downloading. - calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \ - fail "failed to calculate md5sum" - if [[ "$calculated_md5" == "$CIDL_MD5" ]]; then + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" == "$CIDL_SHA256" ]]; then DO_DOWNLOAD="false" fi fi @@ -72,12 +73,12 @@ function main do_download_curl "$CIDL_URL" "$TARBALL_FILE" || \ fail "failed to download file" - # Verify the md5sum. - calculated_md5="$($CIDL_MD5FUNC "$TARBALL_FILE")" || \ - fail "failed to calculate md5sum" - if [[ "$calculated_md5" != "$CIDL_MD5" ]]; then - fail "md5sum mismatch \ - (expected $CIDL_MD5, found $calculated_md5)" + # Verify the sha256sum. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" != "$CIDL_SHA256" ]]; then + fail "sha256sum mismatch \ + (expected $CIDL_SHA256, found $calculated_sha256)" fi fi @@ -105,24 +106,21 @@ function configure_os darwin*) CIDL_BUILD="darwin-10.9-amd64" CIDL_SUFFIX="tgz" - CIDL_MD5="$CIDL_MD5_DARWIN" - CIDL_MD5FUNC="do_md5" + CIDL_SHA256="$CIDL_SHA256_DARWIN" CIDL_URL_BASE="$CIDL_URL_COCKROACH" CIDL_ASSEMBLE="do_assemble_official" ;; linux-gnu*) CIDL_BUILD="linux-amd64" CIDL_SUFFIX="tgz" - CIDL_MD5="$CIDL_MD5_LINUX" - CIDL_MD5FUNC="do_md5sum" + CIDL_SHA256="$CIDL_SHA256_LINUX" CIDL_URL_BASE="$CIDL_URL_COCKROACH" CIDL_ASSEMBLE="do_assemble_official" ;; solaris*) CIDL_BUILD="illumos" CIDL_SUFFIX="tar.gz" - CIDL_MD5="$CIDL_MD5_ILLUMOS" - CIDL_MD5FUNC="do_md5sum" + CIDL_SHA256="$CIDL_SHA256_ILLUMOS" CIDL_URL_BASE="$CIDL_URL_ILLUMOS" CIDL_ASSEMBLE="do_assemble_illumos" ;; @@ -143,14 +141,9 @@ function do_download_curl curl --silent --show-error --fail --location --output "$2" "$1" } -function do_md5 +function do_sha256sum { - md5 < "$1" -} - -function do_md5sum -{ - md5sum < "$1" | awk '{print $1}' + sha256sum < "$1" | awk '{print $1}' } function do_untar diff --git a/tools/cockroachdb_checksums b/tools/cockroachdb_checksums index 50e873100f..20b6e237f8 100644 --- a/tools/cockroachdb_checksums +++ b/tools/cockroachdb_checksums @@ -1,3 +1,3 @@ -CIDL_MD5_DARWIN="2db972c254b4e3b599e12110520178b5" -CIDL_MD5_LINUX="8c3170883e0a0be1a34b44090c067a8c" -CIDL_MD5_ILLUMOS="d8999aff364e5d70f226e139fda724a3" +CIDL_SHA256_DARWIN="1ca69e0911af11a73305c3c6f4650b912d70754900b5bf7b80a1d361efe36561" +CIDL_SHA256_LINUX="24c321820e7ee45fa07fe91ac138befe13ad860e41c6ed595ce58823205ff4a9" +CIDL_SHA256_ILLUMOS="f151714ba3a6e02caaaa59727482c36085e60d6bd2fa963938e9a3d8c8a77088"