Skip to content

Commit

Permalink
some normalization
Browse files Browse the repository at this point in the history
Signed-off-by: Marcos Yacob <[email protected]>
  • Loading branch information
MarcosDY committed Sep 24, 2024
1 parent 89c3913 commit f975f8e
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 189 deletions.
Original file line number Diff line number Diff line change
@@ -1,76 +1,71 @@
#!/bin/bash

# Check at most 30 times (with one second in between) that the server has
# successfully synced down the local authorities.
# Constants
MAXCHECKS=30
RETRY_DELAY=1

check-x509Authorities() {
local expectedBundle=$1
local containerName=$2

local difference=""

while [[ $RETRY_COUNT -lt $MAX_RETRIES ]]; do
log-info "checking for x509 authorities propagation ($i of $MAXCHECKS max)..."

x509Authorities=$(docker compose exec -T ${containerName} \
/opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)

difference=$(diff <(echo $expectedBundle) <(echo $x509Authorities))
if [[ $difference == "" ]];
then
break
else
RETRY_COUNT=$((RETRY_COUNT + 1))
log-debug "x509 authorities not propagated on ${containerName}, retrying in $RETRY_DELAY seconds... ($RETRY_COUNT/$MAX_RETRIES)"
log-debug "difference: $difference"

sleep "${RETRY_DELAY}"
fi

# Fail if retries exceed the maximum
if [[ $RETRY_COUNT -eq $MAX_RETRIES ]]; then
fail-now "Expected bundle: $expectedBundle \n got: $x509Authorities \n difference: $difference"
fi
done
# Function to check x509 authorities propagation
check-x509-authorities() {
local expected_bundle=$1
local container_name=$2
local retry_count=0

while [[ $retry_count -lt $MAXCHECKS ]]; do
log-info "Checking for x509 authorities propagation ($retry_count of $MAXCHECKS max)..."

x509_authorities=$(docker compose exec -T ${container_name} \
/opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)

if diff <(echo "$expected_bundle") <(echo "$x509_authorities") &>/dev/null; then
break
else
retry_count=$((retry_count + 1))
log-debug "x509 authorities not propagated on ${container_name}, retrying in $RETRY_DELAY seconds... ($retry_count/$MAXCHECKS)"
sleep "${RETRY_DELAY}"
fi

# Fail if retries exceed the maximum
if [[ $retry_count -eq $MAXCHECKS ]]; then
fail-now "Expected bundle: $expected_bundle got: $x509_authorities"
fi
done
}

# Initial check for x509 authorities in root-server
x509Authorities=$(docker compose exec -T root-server \
/opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)
x509_authorities=$(docker compose exec -T root-server \
/opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)

amountBundles=$(echo $x509Authorities | jq length)
amount_bundles=$(echo "$x509_authorities" | jq length)

# Ensure only one bundle is present at the start
if [[ $amountBundles -ne 1 ]]; then
fail-now "Only one bundle expected at start"
if [[ $amount_bundles -ne 1 ]]; then
fail-now "Only one bundle expected at start"
fi

# Check x509 authorities propagation across all servers
for server in intermediateA-server intermediateB-server leafA-server leafB-server; do
check-x509Authorities "$x509Authorities" "$server"
check-x509-authorities "$x509_authorities" "$server"
done

# Prepare authority
preparedAuthorityID=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server localauthority x509 prepare -output json | jq -r .prepared_authority.authority_id)
prepared_authority_id=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server localauthority x509 prepare -output json | jq -r .prepared_authority.authority_id)

# Verify that the prepared authority is logged
searching="X509 CA prepared.*local_authority_id=${preparedAuthorityID}"
searching="X509 CA prepared.|local_authority_id=${prepared_authority_id}"
check-log-line root-server "$searching"

# Check for updated x509 authorities in root-server
x509Authorities=$(docker compose exec -T root-server \
/opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)
amountBundles=$(echo $x509Authorities | jq length)
x509_authorities=$(docker compose exec -T root-server \
/opt/spire/bin/spire-server bundle show -output json | jq '.x509_authorities' -c)
amount_bundles=$(echo "$x509_authorities" | jq length)

# Ensure two bundles are present after preparation
if [[ $amountBundles -ne 2 ]]; then
fail-now "Two bundles expected after prepare"
if [[ $amount_bundles -ne 2 ]]; then
fail-now "Two bundles expected after prepare"
fi

# Check x509 authorities propagation across all servers again
for server in intermediateA-server intermediateB-server leafA-server leafB-server; do
check-x509Authorities "$x509Authorities" "$server"
check-x509-authorities "$x509_authorities" "$server"
done

Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
#!/bin/bash

preparedauthority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server \
localauthority x509 show -output json | jq .prepared.authority_id -r)
# Fetch the prepared authority ID
prepared_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server \
localauthority x509 show -output json | jq -r .prepared.authority_id) || fail-now "Failed to fetch prepared authority ID"

activatedauthority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server \
localauthority x509 activate -authorityID ${preparedauthority} -output json | jq .activated_authority.authority_id)
# Activate the authority
activated_authority=$(docker compose exec -t -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server \
localauthority x509 activate -authorityID "${prepared_authority}" \
-output json | jq -r .activated_authority.authority_id) || fail-now "Failed to activate authority"

log-info "Activated authority: ${activateAuthority}"

searching="X509 CA activated*local_authority_id=${preparedAuthority}"
check-log-line root-server $searching
check-log-line root-server "Successfully rotated X.509 CA"
log-info "Activated authority: ${activated_authority}"

# Check logs for specific lines
check-log-line root-server "X509 CA activated|local_authority_id=${prepared_authority}"
check-log-line root-server "Successfully rotated X\.509 CA"
Original file line number Diff line number Diff line change
Expand Up @@ -3,100 +3,69 @@
NUMCHECKS=15
CHECKINTERVAL=2

verifyX509SVID() {
local agent=$1
local expectedAuthority=$2

for ((i=1;i<=NUMCHECKS;i++)); do
log-info "checking for SVID rotation ok $agent ($i of $MAXCHECKS max)..."
# Write svid on disk
docker compose exec -u 1001 -T $agent \
/opt/spire/bin/spire-agent api fetch x509 \
-socketPath /opt/spire/sockets/workload_api.sock \
-write /tmp || fail-now "x509-SVID check failed"

# Copy SVID
docker cp $(docker compose ps -q $1):/tmp/svid.0.pem - | docker cp - $(docker compose ps -q $2):/opt/

docker compose exec -u 1001 -T $2 \
/opt/spire/bin/spire-agent api fetch x509 \
-socketPath /opt/spire/sockets/workload_api.sock \
-write /tmp || fail-now "x509-SVID check failed"

docker compose exec -T $2 openssl verify -verbose -CAfile /tmp/bundle.0.pem -untrusted /opt/svid.0.pem /opt/svid.0.pem

sleep "${CHECKINTERVAL}"
done
check-logs() {
local component=$1
shift
for log in "$@"; do
check-log-line "$component" "$log"
done
}


# Function to check log lines
check_logs() {
local component=$1
shift
for log in "$@"; do
check-log-line ${component} "${log}"
done
}


# Fetch old authority ID
oldAuthority=$(docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server \
localauthority x509 show -output json | jq .old.authority_id -r)
localauthority x509 show -output json | jq .old.authority_id -r) || fail-now "Failed to fetch old authority ID"

# Taint the old authority
docker compose exec -T -e SPIRE_SERVER_FFLAGS=forced_rotation root-server \
/opt/spire/bin/spire-server \
localauthority x509 taint -authorityID ${oldAuthority} -output json | jq .tainted_authority.authority_id
localauthority x509 taint -authorityID "${oldAuthority}" -output json | jq .tainted_authority.authority_id || fail-now "Failed to taint old authority"

# Root server logs
check_logs root-server \
check-logs root-server \
"X\.509 authority tainted successfully|local_authority_id=${oldAuthority}" \
"Server SVID signed using a tainted authority, forcing rotation of the Server SVID"

# Root agent logs
check_logs root-agent \
check-logs root-agent \
"New tainted X.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=3" \
"Agent SVID is tainted by a root authority, forcing rotation"


# Verify workloads are rotated

# Intermediate A server and agent logs
check_logs intermediateA-server \
check-logs intermediateA-server \
"Current root CA is signed by a tainted upstream authority, preparing rotation" \
"Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
check_logs intermediateA-agent \
"New tainted X.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=2" \
check-logs intermediateA-agent \
"New tainted X\.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X.509 authorities|count=2" \
"Agent SVID is tainted by a root authority, forcing rotation"

# Intermediate B server and agent logs
check_logs intermediateB-server \
check-logs intermediateB-server \
"Current root CA is signed by a tainted upstream authority, preparing rotation" \
"Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
check_logs intermediateB-agent \
"New tainted X.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X.509 authorities|count=2" \
check-logs intermediateB-agent \
"New tainted X\.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=2" \
"Agent SVID is tainted by a root authority, forcing rotation"

# Leaf A server and agent logs
check_logs leafA-server \
check-logs leafA-server \
"Current root CA is signed by a tainted upstream authority, preparing rotation" \
"Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
check_logs leafA-agent \
check-logs leafA-agent \
"New tainted X.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X.509 authorities|count=1" \
"Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=1" \
"Agent SVID is tainted by a root authority, forcing rotation"

# Leaf B server and agent logs
check_logs leafB-server \
check-logs leafB-server \
"Current root CA is signed by a tainted upstream authority, preparing rotation" \
"Server SVID signed using a tainted authority, forcing rotation of the Server SVID"
check_logs leafB-agent \
check-logs leafB-agent \
"New tainted X.509 authorities found|subject_key_ids=${oldAuthority}" \
"Scheduled rotation for SVID entries due to tainted X.509 authorities|count=1" \
"Scheduled rotation for SVID entries due to tainted X\.509 authorities|count=1" \
"Agent SVID is tainted by a root authority, forcing rotation"

Original file line number Diff line number Diff line change
Expand Up @@ -3,40 +3,43 @@
MAX_RETRIES=10
RETRY_DELAY=2 # seconds between retries

check_tainted_authorities() {
fetch-x509-authorities() {
local server=$1
docker compose exec -T "$server" /opt/spire/bin/spire-server bundle show -output json | jq .x509_authorities
}

verify-svid() {
local agent=$1
local agent_dir=$2

docker compose exec -u 1001 -T "$agent" \
/opt/spire/bin/spire-agent api fetch x509 \
-socketPath /opt/spire/sockets/workload_api.sock \
-write /tmp || fail-now "x509-SVID check failed for $agent"

docker compose exec -T "$agent" \
openssl verify -verbose -CAfile /opt/spire/conf/agent/non-tainted.pem \
-untrusted /tmp/svid.0.pem /tmp/svid.0.pem
}

check-tainted-authorities() {
local server=$1
local agent=$2
local agent_dir=$3

x509Authorities=$(docker compose exec -T "$server" \
/opt/spire/bin/spire-server \
bundle show -output json | jq .x509_authorities)
x509Authorities=$(fetch-x509-authorities "$server")

taintedFound=$(echo "$x509Authorities" | jq '.[] | select(.tainted == true)')
if [[ -z "$taintedFound" ]]; then
fail-now "Tainted authority expected"
fi
taintedFound=$(echo "$x509Authorities" | jq '.[] | select(.tainted == true)') || fail-now "Tainted authority not found"

nonTaintedFound=$(echo "$x509Authorities" | jq '.[] | select(.tainted == false)')
if [[ -z "$nonTaintedFound" ]]; then
fail-now "Non-tainted authority expected"
fi
nonTaintedFound=$(echo "$x509Authorities" | jq '.[] | select(.tainted == false)') || fail-now "Non-tainted authority not found"

echo "$nonTaintedFound" | jq -r .asn1 | base64 -d | openssl x509 -inform der > "$agent_dir/agent/non-tainted.pem"

RETRY_COUNT=0

while [[ $RETRY_COUNT -lt $MAX_RETRIES ]]; do
docker compose exec -u 1001 -T "$agent" \
/opt/spire/bin/spire-agent api fetch x509 \
-socketPath /opt/spire/sockets/workload_api.sock \
-write /tmp || fail-now "x509-SVID check failed for $agent"
verify-svid "$agent" "$agent_dir"

docker compose exec -T "$agent" \
openssl verify -verbose -CAfile /opt/spire/conf/agent/non-tainted.pem \
-untrusted /tmp/svid.0.pem /tmp/svid.0.pem

# Check the exit status of the verify command
if [ $? -eq 0 ]; then
log-info "SVID rotated"
break
Expand All @@ -46,25 +49,23 @@ check_tainted_authorities() {
sleep $RETRY_DELAY
fi

# Fail if retries exceed the maximum
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
fail-now "Certificate verification failed after $MAX_RETRIES attempts."
fi
done
}

# Root
check_tainted_authorities "root-server" "root-agent" "root"
check-tainted-authorities "root-server" "root-agent" "root"

# IntermediateA
check_tainted_authorities "intermediateA-server" "intermediateA-agent" "intermediateA"
check-tainted-authorities "intermediateA-server" "intermediateA-agent" "intermediateA"

# IntermediateB
check_tainted_authorities "intermediateB-server" "intermediateB-agent" "intermediateB"
check-tainted-authorities "intermediateB-server" "intermediateB-agent" "intermediateB"

# LeafA
check_tainted_authorities "leafA-server" "leafA-agent" "leafA"
check-tainted-authorities "leafA-server" "leafA-agent" "leafA"

# LeafB
check_tainted_authorities "leafB-server" "leafB-agent" "leafB"

check-tainted-authorities "leafB-server" "leafB-agent" "leafB"
Loading

0 comments on commit f975f8e

Please sign in to comment.