diff --git a/scripts/performance/README.md b/scripts/performance/README.md new file mode 100644 index 000000000..0f5c6f490 --- /dev/null +++ b/scripts/performance/README.md @@ -0,0 +1,4 @@ +Performance Scripts +=================== + +This directory is a collection of scripts for exercising rekor's performance. diff --git a/scripts/performance/index-storage/README.md b/scripts/performance/index-storage/README.md new file mode 100644 index 000000000..118037751 --- /dev/null +++ b/scripts/performance/index-storage/README.md @@ -0,0 +1,40 @@ +Rekor Performance Tester +======================== + +Scripts to repeatably gather performance metrics for index storage insertion and +retrieval in rekor. + +Usage +----- + +Use terraform to set up the services in GCP: + +``` +cd terraform +terraform init +terraform plan +terraform apply +``` + +Copy or clone this repository on to the bastion VM that terraform instantiates. +Run this script from there: + +``` +export INSERT_RUNS= # The number of inserts to perform and measure. This doesn't need to be terribly high. +export SEARCH_ENTRIES= # The number of entries to upload to the database out of band to search against. This should be sufficiently high to represent a real database. +export INDEX_BACKEND= # The index backend to test against +export REGION= # The GCP region where the rekor services are deployed +./index-performance.sh +``` + +On the first run, `indices.csv` will be populated with fake search entries, +which will take a while depending on how big $SEARCH_ENTRIES is. This only +happens once as long as indices.csv is not removed. + +Run `index-performance.sh` against each backend. Then plot the results: + +``` +./plot.sh +``` + +Copy the resulting `graph.png` back to your local host. diff --git a/scripts/performance/index-storage/index-performance.sh b/scripts/performance/index-storage/index-performance.sh new file mode 100755 index 000000000..d6fcc929e --- /dev/null +++ b/scripts/performance/index-storage/index-performance.sh @@ -0,0 +1,385 @@ +#!/bin/bash +# +# Copyright 2024 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit + +cleanup() { + code=$? + if [ $code -ne 0 ] ; then + echo "An error occurred. Waiting 30 seconds to start cleanup, press ^C to cancel cleanup." + sleep 30 + fi + $@ + exit $code +} + +INSERT_RUNS=${INSERT_RUNS:-1000} +SEARCH_ENTRIES=${SEARCH_ENTRIES:-100000} +INDEX_BACKEND=${INDEX_BACKEND:-redis} +REGION=${REGION:-us-west1} + +setup_bastion() { + echo "Configuring the bastion..." + sudo apt install kubernetes-client google-cloud-sdk-gke-gcloud-auth-plugin git redis-tools gnuplot prometheus minisign jq -y + if ! which hyperfine >/dev/null ; then + local tag=$(curl -H "Accept: application/json" -L https://github.com/sharkdp/hyperfine/releases/latest | jq -r .tag_name) + wget -O /tmp/hyperfine_${tag:1}_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/${tag}/hyperfine_${tag:1}_amd64.deb + sudo dpkg -i /tmp/hyperfine_${tag:1}_amd64.deb + fi + if ! which helm >/dev/null ; then + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + fi + if ! which rekor-cli >/dev/null ; then + wget -O /tmp/rekor-cli-linux-amd64 https://github.com/sigstore/rekor/releases/latest/download/rekor-cli-linux-amd64 + sudo install -m 0755 /tmp/rekor-cli-linux-amd64 /usr/local/bin/rekor-cli + fi + gcloud auth print-access-token >/dev/null 2>&1 || gcloud auth login + gcloud container clusters get-credentials rekor --region $REGION +} + +setup_rekor() { + echo "Setting up rekor..." + helm repo add sigstore https://sigstore.github.io/helm-charts + helm repo update + + sha=$(git ls-remote https://github.com/sigstore/rekor HEAD | awk '{print substr($1, 1, 7)}') + cat >values.yaml <index-values.yaml <index-values.yaml < patch.yaml </dev/null + rm -rf prometheus/metrics2 + cat >prometheus/prometheus.yml <prometheus/prom.log 2>&1 & + export PROM_PID=$! +} + +# Upload $INSERT_RUNS rekords of $INSERT_RUNS artifacts signed by 1 key +insert() { + echo "Inserting entries..." + local N=$INSERT_RUNS + # Create N artifacts with different contents + export DIR=$(mktemp -d) + for i in $(seq 1 $N) ; do + echo hello${i} > ${DIR}/blob${i} + done + # Create a signing key + minisign -G -p $DIR/user1@example.com.pub -s $DIR/user1@example.com.key -W >/dev/null + + echo "Signing $N artifacts with 1 key" + user=user1@example.com + local batch=0 + while [ $batch -lt $N ] ; do + for i in $(seq 1 100) ; do + let id=$batch+$i + if [ $id -gt $N ] ; then + break + fi + sig=${DIR}/$(uuidgen).asc + ( + minisign -S -x $sig -s $DIR/$user.key -m ${DIR}/blob${id} + rekor-cli upload --rekor_server $REKOR_URL --signature $sig --public-key ${DIR}/${user}.pub --artifact ${DIR}/blob${id} --pki-format=minisign + ) & + done + wait $(jobs -p | grep -v $PROM_PID) + let batch+=100 + done + + rm -rf $DIR +} + +query_inserts() { + echo "Getting metrics for inserts..." + count=null + + # may need to wait for the data to be scraped + tries=0 + until [ "${count}" != "null" ] ; do + sleep 1 + count=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_count{success="true"}' | jq -r .data.result[0].value[1]) + let 'tries+=1' + if [ $tries -eq 6 ] ; then + echo "count query failed, here is the raw result:" + set -x + curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_count{success="true"}' + set +x + echo + exit 1 + fi + done + + avg=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_sum{success="true"}/rekor_index_storage_latency_summary_count{success="true"}' | jq -r .data.result[0].value[1]) + + if [ "${avg}" == "null" ] ; then + echo "avg query failed, here is the raw result:" + set -x + curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_index_storage_latency_summary_sum{success="true"}/rekor_index_storage_latency_summary_count{success="true"}' + set +x + echo + exit 1 + fi + + echo "Insert latency: ${avg} (average over ${count} inserts)" + results=${INDEX_BACKEND}.dat + if [ "$INDEX_BACKEND" == "redis" ] ; then + x=1 + else + x=0 + fi + # output to gnuplot data set + echo "$x \"${INDEX_BACKEND} inserts\n(${count})\" $avg" > $results +} + +upload() { + echo "Uploading entries..." + N=$SEARCH_ENTRIES + + if [ ! -f indices.csv ] ; then + echo "Generating $N * 2 entries. This may take a while..." + # N artifacts, 1 user + for i in $(seq 1 $N) ; do + uuid=$(dbus-uuidgen) + echo user1@example.com,$uuid >> indices.csv + sha=$(echo $i | sha256sum | cut -d ' ' -f 1) + echo sha256:$sha,$uuid >> indices.csv + done + + # 1 artifact, N users + sha=$(echo 1 | sha256sum | cut -d ' ' -f 1) + for i in $(seq 2 $N) ; do + uuid=$(dbus-uuidgen) + echo user${i}@example.com,$uuid >> indices.csv + echo sha256:$sha,$uuid >> indices.csv + done + fi + + if [ "${INDEX_BACKEND}" == "redis" ] ; then + local dbsize=$(redis-cli -h $REDIS_IP dbsize | cut -d ' ' -f 2) + let wantsize=$SEARCH_ENTRIES*2 + if [ ! $dbsize -ge $wantsize ] ; then + echo "Uploading entries into redis..." + while read LINE ; do + key=$(echo $LINE | cut -d',' -f1) + val=$(echo $LINE | cut -d',' -f2) + printf "*3\r\n\$5\r\nLPUSH\r\n\$${#key}\r\n${key}\r\n\$${#val}\r\n${val}\r\n" + done < indices.csv | redis-cli -h $REDIS_IP --pipe + fi + else + local dbsize=$(mysql -h $MYSQL_IP -P 3306 -utrillian -p${MYSQL_PASS} -D trillian -e "SELECT COUNT(*) FROM EntryIndex" --vertical | tail -1 | cut -d ' ' -f 2) + let wantsize=$SEARCH_ENTRIES*4 + if [ ! $dbsize -ge $wantsize ] ; then + echo "Uploading entries into mysql..." + mysql -h $MYSQL_IP -P 3306 -utrillian -p${MYSQL_PASS} -D trillian -e "CREATE TABLE IF NOT EXISTS EntryIndex ( + PK BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + EntryKey varchar(512) NOT NULL, + EntryUUID char(80) NOT NULL, + PRIMARY KEY(PK), + UNIQUE(EntryKey, EntryUUID) + ); + LOAD DATA LOCAL INFILE './indices.csv' + INTO TABLE EntryIndex + FIELDS TERMINATED BY ',' + LINES TERMINATED BY '\n' (EntryKey, EntryUUID);" + fi + fi +} + +search() { + echo "Running search requests..." + sumblob1=$(echo 1 | sha256sum | cut -d ' ' -f1) + sumblob2=$(echo 2 | sha256sum | cut -d ' ' -f1) + sumblobnone=$(echo none | sha256sum | cut -d ' ' -f1) + # Search for entries using public key user1@example.com (should be many), user2@example.com (should be few), notreal@example.com (should be none) + hyperfine --style basic --warmup 10 --ignore-failure --parameter-list email user1@example.com,user2@example.com,notreal@example.com "rekor-cli search --rekor_server $REKOR_URL --email {email}" + # Search for entries using the sha256 sum of blob1 (should be many), blob2 (should be few), blobnone (should be none) + hyperfine --style basic --warmup 10 --ignore-failure --parameter-list sha ${sumblob1},${sumblob2},${sumblobnone} "rekor-cli search --rekor_server $REKOR_URL --sha sha256:{sha}" + # Search for entries using public key user1@example.com/user2@example.com/notreal@example.com OR/AND sha256 sum of blob1/blob2/blobnone + hyperfine --style basic --warmup 10 --ignore-failure --parameter-list email user1@example.com,user2@example.com,notreal@example.com \ + --parameter-list sha ${sumblob1},${sumblob2},${sumblobnone} \ + --parameter-list operator or,and \ + "rekor-cli search --rekor_server $REKOR_URL --email {email} --sha sha256:{sha} --operator {operator}" +} + +query_search() { + echo "Getting metrics for searches..." + count=null + # may need to wait for the data to be scraped + tries=0 + until [ "${count}" != "null" ] ; do + sleep 1 + count=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}' | jq -r .data.result[0].value[1]) + let 'tries+=1' + if [ $tries -eq 6 ] ; then + echo "count query failed, here is the raw result:" + set -x + curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}' + set +x + echo + fi + done + + avg=$(curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_sum{path="/api/v1/index/retrieve"}/rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}' | jq -r .data.result[0].value[1]) + if [ "${avg}" == "null" ] ; then + echo "avg query failed, here is the raw result:" + set -x + curl -s http://localhost:9090/api/v1/query --data-urlencode 'query=rekor_api_latency_summary_sum{path="/api/v1/index/retrieve"}/rekor_api_latency_summary_count{path="/api/v1/index/retrieve"}' + set +x + echo + fi + + echo "Search latency: ${avg} (average over ${count} searches)" + results=${INDEX_BACKEND}.dat + if [ "$INDEX_BACKEND" == "redis" ] ; then + x=3 + else + x=2 + fi + # output to gnuplot data set + echo "$x \"${INDEX_BACKEND} searches\n(${count})\" $avg" >> $results +} + +reset() { + echo "Resetting data..." + if [ "${INDEX_BACKEND}" == "redis" ] ; then + redis-cli -h $REDIS_IP flushall + else + mysql -h $MYSQL_IP -P 3306 -utrillian -p${MYSQL_PASS} -D trillian -e 'DELETE FROM EntryIndex;' + fi + kubectl -n rekor-system rollout restart deployment rekor-server +} + +if [ "${BASH_SOURCE[0]}" == "${0}" ]; then + if [ "${INDEX_BACKEND}" != "redis" -a "${INDEX_BACKEND}" != "mysql" ] ; then + echo '$INDEX_BACKEND must be either redis or mysql.' + exit 1 + fi + + if [ "${INDEX_BACKEND}" == "mysql" -a "${MYSQL_PASS}" == "" ] ; then + echo '$MYSQL_PASS must be set when $INDEX_BACKEND is mysql.' + echo 'The trillian mysql user password can be found from your terraform host using `terraform output -json | jq -r .mysql_pass.value`.' + exit 1 + fi + + echo "Gathering insertion and retrieval metrics for index backend [${INDEX_BACKEND}]." + + setup_bastion + + setup_rekor + + if [ -n "$RESET" ] ; then + reset + fi + + setup_prometheus + cleanup_prom() { + echo "Cleaning up prometheus..." + pkill -x prometheus + } + trap 'cleanup cleanup_prom' EXIT + + insert + + query_inserts + + upload + + search + + query_search +fi diff --git a/scripts/performance/index-storage/plot.gp b/scripts/performance/index-storage/plot.gp new file mode 100755 index 000000000..a6d2cf767 --- /dev/null +++ b/scripts/performance/index-storage/plot.gp @@ -0,0 +1,35 @@ +#!/usr/bin/gnuplot +# +# Copyright 2024 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set term png +set output "graph.png" + +set style line 1 lc rgb "#40a7db" +set style line 2 lc rgb "#b38df0" +set yrange [0:] +set style fill solid +set boxwidth 0.5 +set ylabel "Latency (nanoseconds)" +set xlabel "Operation\nAverage over (N operations),\nsearches across " . entries . " entries" +set format y '%.0f' +set bmargin 6 +set grid y +set tics font "sans,10" + +plot "results.dat" every ::0::1 using 1:3:xtic(2) with boxes linestyle 1 notitle, \ + "results.dat" every ::0::1 using 1:($3+1000000):(sprintf('%3.2f', $3)) with labels font "sans,10" notitle, \ + "results.dat" every ::2::3 using 1:3:xtic(2) with boxes linestyle 2 notitle, \ + "results.dat" every ::2::3 using 1:($3+1000000):(sprintf('%3.2f', $3)) with labels font "sans,10" notitle diff --git a/scripts/performance/index-storage/plot.sh b/scripts/performance/index-storage/plot.sh new file mode 100755 index 000000000..70011a6bd --- /dev/null +++ b/scripts/performance/index-storage/plot.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2024 The Sigstore Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cat redis.dat mysql.dat > results.dat +entries=$(wc -l indices.csv | cut -d ' ' -f 1) +gnuplot -e "entries='$entries'" plot.gp + +echo "Results output to graph.png." diff --git a/scripts/performance/index-storage/terraform/main.tf b/scripts/performance/index-storage/terraform/main.tf new file mode 100644 index 000000000..a2255263b --- /dev/null +++ b/scripts/performance/index-storage/terraform/main.tf @@ -0,0 +1,126 @@ +// Copyright 2024 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +provider "google" { + project = var.project + zone = var.zone + region = var.region +} + +module "network" { + source = "git::https://github.com/sigstore/scaffolding.git//terraform/gcp/modules/network?ref=feced1f5bf972edcf37282e80a2b593474a7d807" + + region = var.region + project_id = var.project + cluster_name = "rekor" +} + +module "bastion" { + source = "git::https://github.com/sigstore/scaffolding.git//terraform/gcp/modules/bastion?ref=feced1f5bf972edcf37282e80a2b593474a7d807" + + project_id = var.project + region = var.region + zone = var.zone + network = module.network.network_name + subnetwork = module.network.subnetwork_self_link + tunnel_accessor_sa = var.tunnel_accessor_sa + + depends_on = [ + module.network, + ] +} + +module "mysql" { + source = "git::https://github.com/sigstore/scaffolding.git//terraform/gcp/modules/mysql?ref=feced1f5bf972edcf37282e80a2b593474a7d807" + + project_id = var.project + region = var.region + cluster_name = "rekor" + database_version = "MYSQL_8_0" + availability_type = "ZONAL" + network = module.network.network_self_link + instance_name = "rekor-perf-tf" + require_ssl = false + + depends_on = [ + module.network + ] +} + +module "gke_cluster" { + source = "git::https://github.com/sigstore/scaffolding.git//terraform/gcp/modules/gke_cluster?ref=feced1f5bf972edcf37282e80a2b593474a7d807" + + region = var.region + project_id = var.project + cluster_name = "rekor" + network = module.network.network_self_link + subnetwork = module.network.subnetwork_self_link + cluster_secondary_range_name = module.network.secondary_ip_range.0.range_name + services_secondary_range_name = module.network.secondary_ip_range.1.range_name + cluster_network_tag = "" + bastion_ip_address = module.bastion.ip_address + + depends_on = [ + module.network, + module.bastion, + ] +} + +module "rekor" { + source = "git::https://github.com/sigstore/scaffolding.git//terraform/gcp/modules/rekor?ref=feced1f5bf972edcf37282e80a2b593474a7d807" + + region = var.region + project_id = var.project + cluster_name = "rekor" + + enable_attestations = false + + redis_cluster_memory_size_gb = "16" + + network = module.network.network_self_link + dns_zone_name = var.dns_zone + dns_domain_name = var.dns_domain + + depends_on = [ + module.network, + module.gke_cluster + ] +} + +module "oslogin" { + source = "git::https://github.com/sigstore/scaffolding.git//terraform/gcp/modules/oslogin?ref=feced1f5bf972edcf37282e80a2b593474a7d807" + + project_id = var.project + count = 1 + oslogin = { + enabled = true + enabled_with_2fa = false + } + instance_os_login_members = { + bastion = { + instance_name = module.bastion.name + zone = module.bastion.zone + members = var.oslogin_members + } + } + + depends_on = [ + module.bastion, + ] +} + +output "mysql_pass" { + value = module.mysql.mysql_pass + sensitive = true +} diff --git a/scripts/performance/index-storage/terraform/variables.tf b/scripts/performance/index-storage/terraform/variables.tf new file mode 100644 index 000000000..c2ac87f4d --- /dev/null +++ b/scripts/performance/index-storage/terraform/variables.tf @@ -0,0 +1,41 @@ +// Copyright 2024 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +variable "project" { + type = string +} + +variable "region" { + default = "us-west1" +} + +variable "zone" { + default = "us-west1-c" +} + +variable "tunnel_accessor_sa" { + type = string +} + +variable "oslogin_members" { + type = list(string) +} + +variable "dns_zone" { + type = string +} + +variable "dns_domain" { + type = string +}