diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 80977efc8..b9bfc0fcd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -146,6 +146,8 @@ jobs: run: ./tests/cleanup-index-test.sh env: INDEX_BACKEND: redis + - name: Copy index test + run: ./tests/copy-index-test.sh - name: Upload logs if they exist uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 if: failure() @@ -163,7 +165,7 @@ jobs: run: sudo add-apt-repository ppa:savoury1/minisign && sudo apt-get update && sudo apt-get install minisign - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Docker Build - run: docker-compose build + run: docker compose build - name: Extract version of Go to use run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 @@ -186,7 +188,7 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Docker Build - run: docker-compose build + run: docker compose build - name: Extract version of Go to use run: echo "GOVERSION=$(cat Dockerfile|grep golang | awk ' { print $2 } ' | cut -d '@' -f 1 | cut -d ':' -f 2 | uniq)" >> $GITHUB_ENV - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 diff --git a/cmd/cleanup-index/main.go b/cmd/cleanup-index/main.go index 143aeafde..7452fa61e 100644 --- a/cmd/cleanup-index/main.go +++ b/cmd/cleanup-index/main.go @@ -30,6 +30,7 @@ import ( "log" "os" "os/signal" + "strings" "syscall" _ "github.com/go-sql-driver/mysql" @@ -171,6 +172,9 @@ func removeFromRedis(ctx context.Context, redisClient *redis.Client, keys []stri if *dryRun { return nil } + for i, k := range keys { + keys[i] = strings.ToLower(k) + } result, err := redisClient.Del(ctx, keys...).Result() if err != nil { return err diff --git a/cmd/copy-index/main.go b/cmd/copy-index/main.go new file mode 100644 index 000000000..b6414c689 --- /dev/null +++ b/cmd/copy-index/main.go @@ -0,0 +1,224 @@ +// Copyright 2024 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + copy-index is a script to copy indexes from one provider to another. + Currently, only copying from Redis to MySQL is supported. This is useful + when the data already exists in one backend and needs to be migrated to a + new provider. + + To run: + go run cmd/copy-index/main.go --redis-hostname --redis-port \ + --mysql-dsn [--dry-run] +*/ + +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "log" + "os" + "os/signal" + "strconv" + "syscall" + + _ "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + "github.com/redis/go-redis/v9" + "sigs.k8s.io/release-utils/version" +) + +const ( + mysqlWriteStmt = "INSERT IGNORE INTO EntryIndex (EntryKey, EntryUUID) VALUES (:key, :uuid)" + mysqlCreateTableStmt = `CREATE TABLE IF NOT EXISTS EntryIndex ( + PK BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + EntryKey varchar(512) NOT NULL, + EntryUUID char(80) NOT NULL, + PRIMARY KEY(PK), + UNIQUE(EntryKey, EntryUUID) + )` +) + +type redisClient struct { + client *redis.Client + cursor int +} + +type mysqlClient struct { + client *sqlx.DB +} + +var ( + redisHostname = flag.String("redis-hostname", "", "Hostname for Redis application") + redisPort = flag.String("redis-port", "", "Port to Redis application") + redisPassword = flag.String("redis-password", "", "Password for Redis authentication") + redisEnableTLS = flag.Bool("redis-enable-tls", false, "Enable TLS for Redis client") + redisInsecureSkipVerify = flag.Bool("redis-insecure-skip-verify", false, "Whether to skip TLS verification for Redis client or not") + mysqlDSN = flag.String("mysql-dsn", "", "MySQL Data Source Name") + batchSize = flag.Int("batch-size", 10000, "Number of Redis entries to scan per batch (use for testing)") + versionFlag = flag.Bool("version", false, "Print the current version of Copy Index") + dryRun = flag.Bool("dry-run", false, "Dry run - don't actually insert into MySQL") +) + +func main() { + flag.Parse() + + versionInfo := version.GetVersionInfo() + if *versionFlag { + fmt.Println(versionInfo.String()) + os.Exit(0) + } + + if *redisHostname == "" { + log.Fatal("Redis address must be set") + } + if *redisPort == "" { + log.Fatal("Redis port must be set") + } + if *mysqlDSN == "" { + log.Fatal("MySQL DSN must be set") + } + + log.Printf("running copy index Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate) + + mysqlClient, err := getMySQLClient() + if err != nil { + log.Fatalf("creating mysql client: %v", err) + } + redisClient, err := getRedisClient() + if err != nil { + log.Fatalf("creating redis client: %v", err) + } + + err = doCopy(mysqlClient, redisClient) + if err != nil { + log.Fatalf("populating index: %v", err) + } +} + +// getMySQLClient creates a MySQL client. +func getMySQLClient() (*mysqlClient, error) { + dbClient, err := sqlx.Open("mysql", *mysqlDSN) + if err != nil { + return nil, err + } + if err = dbClient.Ping(); err != nil { + return nil, err + } + if _, err = dbClient.Exec(mysqlCreateTableStmt); err != nil { + return nil, err + } + return &mysqlClient{client: dbClient}, nil +} + +// getRedisClient creates a Redis client. +func getRedisClient() (*redisClient, error) { + opts := &redis.Options{ + Addr: fmt.Sprintf("%s:%s", *redisHostname, *redisPort), + Password: *redisPassword, + Network: "tcp", + DB: 0, // default DB + } + // #nosec G402 + if *redisEnableTLS { + opts.TLSConfig = &tls.Config{ + InsecureSkipVerify: *redisInsecureSkipVerify, //nolint: gosec + } + } + return &redisClient{client: redis.NewClient(opts)}, nil +} + +// doCopy pulls search index entries from the Redis database and copies them into the MySQL database. +func doCopy(mysqlClient *mysqlClient, redisClient *redisClient) error { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + var err error + var done bool + var keys []string + for !done { + keys, done, err = redisClient.getIndexKeys(ctx) + if err != nil { + return err + } + for _, k := range keys { + uuids, err := redisClient.getUUIDsForKey(ctx, k) + if err != nil { + return err + } + for _, v := range uuids { + err = mysqlClient.idempotentAddToIndex(ctx, k, v) + if err != nil { + return err + } + } + } + } + fmt.Println("Copy complete") + return nil +} + +// getIndexKeys looks up every key in Redis that is not a checkpoint string. +// It limits the size of the scan to the value of --batch-size and uses the +// returned cursor to keep track of whether the scan is complete. +// It returns a boolean true when the call does not need to be repeated to get more keys. +func (c *redisClient) getIndexKeys(ctx context.Context) ([]string, bool, error) { + result, err := c.client.Do(ctx, "SCAN", c.cursor, "TYPE", "list", "COUNT", *batchSize).Result() // go-redis Scan method does not support TYPE + if err != nil { + return nil, false, err + } + resultList, ok := result.([]any) + if !ok { + return nil, false, fmt.Errorf("unexpected result from Redis SCAN command: %v", result) + } + if len(resultList) != 2 { + return nil, false, fmt.Errorf("unexpected result from Redis SCAN command: %v", resultList) + } + cursor, ok := resultList[0].(string) + if !ok { + return nil, false, fmt.Errorf("could not parse returned cursor from Redis SCAN command: %v", resultList[0]) + } + c.cursor, err = strconv.Atoi(cursor) + if err != nil { + return nil, false, fmt.Errorf("could not parse returned cursor from Redis SCAN command: %v", cursor) + } + keys, ok := resultList[1].([]any) + if !ok { + return nil, false, fmt.Errorf("could not parse returned keys from Redis SCAN command: %v", resultList[1]) + } + keyStrings := make([]string, len(keys)) + for i, k := range keys { + keyStrings[i], ok = k.(string) + if !ok { + return nil, false, fmt.Errorf("could not parse returned keys from Redis SCAN command: %v", k) + } + } + fmt.Printf("Processing %d keys - cursor %d\n", len(keys), c.cursor) + return keyStrings, c.cursor == 0, nil +} + +// getUUIDsForKey returns the list of UUIDs for a given index key. +func (c *redisClient) getUUIDsForKey(ctx context.Context, key string) ([]string, error) { + return c.client.LRange(ctx, key, 0, -1).Result() +} + +// idempotentAddToIndex inserts the given key-value pair into the MySQL search index table. +func (c *mysqlClient) idempotentAddToIndex(ctx context.Context, key, value string) error { + if *dryRun { + return nil + } + _, err := c.client.NamedExecContext(ctx, mysqlWriteStmt, map[string]any{"key": key, "uuid": value}) + return err +} diff --git a/tests/backfill-test.sh b/tests/backfill-test.sh index fa5658bcc..2076999fd 100755 --- a/tests/backfill-test.sh +++ b/tests/backfill-test.sh @@ -33,44 +33,6 @@ source $(dirname "$0")/index-test-utils.sh trap cleanup EXIT -make_entries() { - set -e - # make 10 unique artifacts and sign each once - for i in $(seq 0 9) ; do - minisign -GW -p $testdir/mini${i}.pub -s $testdir/mini${i}.key - echo test${i} > $testdir/blob${i} - minisign -S -s $testdir/mini${i}.key -m $testdir/blob${i} - local rekor_out=$(rekor-cli --rekor_server $REKOR_ADDRESS upload \ - --artifact $testdir/blob${i} \ - --pki-format=minisign \ - --public-key $testdir/mini${i}.pub \ - --signature $testdir/blob${i}.minisig \ - --format json) - local uuid=$(echo $rekor_out | jq -r .Location | cut -d '/' -f 6) - expected_keys["$testdir/mini${i}.pub"]=$uuid - expected_artifacts["$testdir/blob${i}"]=$uuid - done - # double-sign a few artifacts - for i in $(seq 7 9) ; do - set +e - let key_index=$i-5 - set -e - minisign -S -s $testdir/mini${key_index}.key -m $testdir/blob${i} - rekor_out=$(rekor-cli --rekor_server $REKOR_ADDRESS upload \ - --artifact $testdir/blob${i} \ - --pki-format=minisign \ - --public-key $testdir/mini${key_index}.pub \ - --signature $testdir/blob${i}.minisig \ - --format json) - uuid=$(echo $rekor_out | jq -r .Location | cut -d '/' -f 6) - local orig_key_uuid="${expected_keys[${testdir}/mini${key_index}.pub]}" - expected_keys[$testdir/mini${key_index}.pub]="$orig_key_uuid $uuid" - local orig_art_uuid="${expected_artifacts[${testdir}/blob${i}]}" - expected_artifacts[${testdir}/blob${i}]="$orig_art_uuid $uuid" - done - set +e -} - remove_keys() { set -e for i in $@ ; do @@ -88,24 +50,6 @@ remove_keys() { set +e } -search_expect_fail() { - local artifact=$1 - rekor-cli --rekor_server $REKOR_ADDRESS search --artifact $artifact 2>/dev/null - if [ $? -eq 0 ] ; then - echo "Unexpected index found." - exit 1 - fi -} - -search_expect_success() { - local artifact=$1 - rekor-cli --rekor_server $REKOR_ADDRESS search --artifact $artifact 2>/dev/null - if [ $? -ne 0 ] ; then - echo "Unexpected missing index." - exit 1 - fi -} - check_all_entries() { set -e for artifact in "${!expected_artifacts[@]}" ; do diff --git a/tests/copy-index-test.sh b/tests/copy-index-test.sh new file mode 100755 index 000000000..35a3b2a56 --- /dev/null +++ b/tests/copy-index-test.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# +# Copyright 2024 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REKOR_ADDRESS=http://localhost:3000 +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_PASSWORD=test +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=test +MYSQL_PASSWORD=zaphod +MYSQL_DB=test + +testdir=$(mktemp -d) + +declare -A expected_artifacts +declare -A expected_keys + +source $(dirname "$0")/index-test-utils.sh + +trap cleanup EXIT + +make_intoto_entries() { + set -e + for type in intoto:0.0.1 intoto:0.0.2 dsse ; do + rekor-cli --rekor_server $REKOR_ADDRESS upload \ + --type $type \ + --artifact tests/intoto_dsse.json \ + --public-key tests/intoto_dsse.pem \ + --format=json + done + set +e +} + +search_sha_expect_success() { + local sha=$1 + rekor-cli --rekor_server $REKOR_ADDRESS search --sha $sha 2>/dev/null + if [ $? -ne 0 ] ; then + echo "Unexpected missing index." + exit 1 + fi +} + +check_basic_entries() { + set -e + for artifact in "${!expected_artifacts[@]}" ; do + local expected_uuids="${expected_artifacts[$artifact]}" + local sha=$(sha256sum $artifact | cut -d ' ' -f 1) + local actual_uuids=$(rekor-cli --rekor_server $REKOR_ADDRESS search --sha $sha --format json | jq -r .UUIDs[]) + for au in $actual_uuids ; do + local found=0 + for eu in $expected_uuids ; do + if [ "$au" == "$eu" ] ; then + found=1 + break + fi + done + if [ $found -eq 0 ] ; then + echo "Backfill did not add expected artifact $artifact." + exit 1 + fi + done + expected_uuids=($expected_uuids) + local expected_length=${#expected_uuids[@]} + # Check the values of each key for redis so we know there aren't duplicates. + # We don't need to do this for mysql, we'll just go by the total row count. + if [ "$INDEX_BACKEND" == "redis" ] ; then + local actual_length=$(redis_cli LLEN sha256:${sha}) + if [ $expected_length -ne $actual_length ] ; then + echo "Possible dupicate keys for artifact $artifact." + exit 1 + fi + fi + done + for key in "${!expected_keys[@]}" ; do + expected_uuids=${expected_keys[$key]} + actual_uuids=$(rekor-cli --rekor_server $REKOR_ADDRESS search --pki-format minisign --public-key $key --format json | jq -r .UUIDs[]) + for au in $actual_uuids ; do + local found=0 + for eu in $expected_uuids ; do + if [ "$au" == "$eu" ] ; then + found=1 + break + fi + done + if [ $found -eq 0 ] ; then + echo "Backfill did not add expected key $key." + exit 1 + fi + done + local keysha=$(echo -n $(tail -1 $key) | sha256sum | cut -d ' ' -f 1) + expected_uuids=($expected_uuids) + local expected_length=${#expected_uuids[@]} + # Check the values of each key for redis so we know there aren't duplicates. + # We don't need to do this for mysql, we'll just go by the total row count. + if [ "$INDEX_BACKEND" = "redis" ] ; then + local actual_length=$(redis_cli LLEN $keysha) + if [ $expected_length -ne $actual_length ] ; then + echo "Possible dupicate keys for artifact $artifact." + exit 1 + fi + fi + done + set +e +} + +check_intoto_entries() { + local dsse_sha=$(sha256sum tests/intoto_dsse.json | cut -d ' ' -f 1) + local dsse_key_sha=$(echo | cat tests/intoto_dsse.pem - | sha256sum | cut -d ' ' -f 1) + local dsse_payload=$(jq -r .payload tests/intoto_dsse.json | base64 -d | sha256sum) + + search_expect_success tests/intoto_dsse.json + search_sha_expect_success $dsse_sha + search_sha_expect_success $dsse_key_sha + search_sha_expect_success $dsse_payload +} + +run_copy() { + set -e + go run cmd/copy-index/main.go \ + --mysql-dsn "${MYSQL_USER}:${MYSQL_PASSWORD}@tcp(${MYSQL_HOST}:${MYSQL_PORT})/${MYSQL_DB}" \ + --redis-hostname $REDIS_HOST --redis-port $REDIS_PORT --redis-password $REDIS_PASSWORD \ + --batch-size 5 + set +e +} + +export INDEX_BACKEND=redis +docker_up + +make_entries +make_intoto_entries + +check_basic_entries +check_intoto_entries + +export INDEX_BACKEND=mysql +docker-compose stop rekor-server +docker_up + +search_expect_fail tests/intoto_dsse.json # the new index backend should be empty at this point + +run_copy + +check_basic_entries +check_intoto_entries diff --git a/tests/index-test-utils.sh b/tests/index-test-utils.sh index d94cb79aa..814ffe6a3 100644 --- a/tests/index-test-utils.sh +++ b/tests/index-test-utils.sh @@ -14,6 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +docker_compose="docker compose -f docker-compose.yml -f docker-compose.backfill-test.yml" +if ! ${docker_compose} version >/dev/null 2>&1; then + docker_compose="docker-compose -f docker-compose.yml -f docker-compose.backfill-test.yml" +fi + make_entries() { set -e # make 10 unique artifacts and sign each once @@ -55,19 +60,19 @@ make_entries() { cleanup() { rv=$? if [ $rv -ne 0 ] ; then - docker-compose -f docker-compose.yml -f docker-compose.backfill-test.yml logs --no-color > /tmp/docker-compose.log + ${docker_compose} logs --no-color > /tmp/docker-compose.log fi - docker-compose down --remove-orphans + ${docker_compose} down --remove-orphans rm -rf $testdir exit $rv } docker_up () { set -e - docker-compose -f docker-compose.yml -f docker-compose.backfill-test.yml up -d --build + ${docker_compose} up -d --build local count=0 echo "waiting up to 2 min for system to start" - until [ $(docker-compose ps | \ + until [ $(${docker_compose} ps | \ grep -E "(rekor[-_]mysql|rekor[-_]redis|rekor[-_]rekor-server)" | \ grep -c "(healthy)" ) == 3 ]; do @@ -95,3 +100,20 @@ mysql_cli() { set +e } +search_expect_fail() { + local artifact=$1 + rekor-cli --rekor_server $REKOR_ADDRESS search --artifact $artifact 2>/dev/null + if [ $? -eq 0 ] ; then + echo "Unexpected index found." + exit 1 + fi +} + +search_expect_success() { + local artifact=$1 + rekor-cli --rekor_server $REKOR_ADDRESS search --artifact $artifact 2>/dev/null + if [ $? -ne 0 ] ; then + echo "Unexpected missing index." + exit 1 + fi +}