soak.yaml

# name: Soak Test

# permissions: read-all
# on:
#   workflow_dispatch:
#   schedule:
#     - cron: '0 4 * * *' # 12AM EST/9PM PST
#   # push:
#   #   paths-ignore:
#   #   - "**.md"
#   #   - "**.jpg"
#   #   - "**.png"
#   #   - "**.gif"
#   #   - "**.svg"
#   #   - "adr/**"
#   #   - "__mocks__/**"
#   #   - ".npmignore"
#   #   - ".prettierignore"
#   #   - "scripts/**"
#   #   - "hack/**"
#   #   - "_images/**"
#   #   - "**.yaml"
#   #   - "**.toml"
#   #   - "**.json"
#   #   - "docs/**"
#   #   - "CODEOWNERS"
#   #   branches: ["main"]
#   # pull_request:
#   #   branches: ["main"]
#   #   paths-ignore:
#   #   - "**.md"
#   #   - "**.jpg"
#   #   - "**.png"
#   #   - "**.gif"
#   #   - "**.svg"
#   #   - "adr/**"
#   #   - "__mocks__/**"
#   #   - ".npmignore"
#   #   - ".prettierignore"
#   #   - "scripts/**"
#   #   - "hack/**"
#   #   - "_images/**"
#   #   - "**.yaml"
#   #   - "**.toml"
#   #   - "docs/**"
#   #   - "**.json"
#   #   - "CODEOWNERS"
# # refs
# # https://frontside.com/blog/2022-12-12-dynamic-github-action-jobs/
# # https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/

# jobs:
#   pepr-build:
#     name: controller image
#     runs-on: ubuntu-latest
#     steps:
#       - name: Harden Runner
#         uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
#         with:
#           egress-policy: audit

#       - name: clone pepr
#         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
#         with:
#           repository: defenseunicorns/pepr
#           path: pepr

#       - name: "set env: PEPR"
#         run: echo "PEPR=${GITHUB_WORKSPACE}/pepr" >> "$GITHUB_ENV"

#       - name: setup node
#         uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
#         with:
#           node-version: 20
#           cache: "npm"
#           cache-dependency-path: pepr

#       - name: install pepr deps
#         run: |
#           cd "$PEPR"
#           npm ci

#       - name: build pepr image
#         run: |
#           cd "$PEPR"
#           npm run build:image

#       - name: tar pepr image
#         run: |
#           PEPR_TAR="${GITHUB_WORKSPACE}/pepr-img.tar"
#           echo "PEPR_TAR=${PEPR_TAR}" >> "$GITHUB_ENV"
#           docker image save --output "$PEPR_TAR" pepr:dev

#       - name: upload image tar artifact
#         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
#         with:
#           name: pepr-img.tar
#           path: pepr-img.tar
#           retention-days: 1

#   soak-test:
#     name: soak-test
#     runs-on: ubuntu-latest
#     needs:
#       - pepr-build
      
#     steps:
#       - name: Harden Runner
#         uses: step-security/harden-runner@5c7944e73c4c2a096b17a9cb74d65b6c2bbafbde # v2.9.1
#         with:
#           egress-policy: audit

#       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

#       - name: "install k3d"
#         run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"
#         shell: bash

#       - name: dowload image tar artifact
#         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
#         with:
#           name: pepr-img.tar
#           path: ${{ github.workspace }}

#       - name: import pepr image from tar
#         run: |
#           PEPR_TAR="${GITHUB_WORKSPACE}/pepr-img.tar"
#           echo "PEPR_TAR=${PEPR_TAR}" >> "$GITHUB_ENV"
#           docker image load --input "$PEPR_TAR"

#       - name: Create k3d Cluster
#         run: "k3d cluster create"
#         shell: bash

#       - name: Import pepr image into k3d
#         run: "k3d image import pepr:dev -c k3s-default"
#         shell: bash

#       - name: Install istioctl
#         run: |
#           curl -L https://istio.io/downloadIstio | sh -
#         shell: bash

#       - name: Install default profile
#         run: |
#           cd istio*/bin
#           ./istioctl install --set profile=demo -y

#       - name: Set up Kubernetes
#         uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4.0.0
#         with:
#           version: 'latest'

#       - name: Create logs directory
#         run: mkdir -p logs

#       - name: Deploy applications
#         run: |
#           kubectl apply -f hack/auditor.ci.yaml
#           kubectl apply -f hack/soak.ci.yaml

#       - name: Wait for 15 minutes before starting pod checks
#         run: |
#           sleep 10s
#           kubectl wait --for=condition=ready -n istio-system pod -l istio=pilot
#           kubectl wait --for=condition=ready -n istio-system pod -l app=istio-ingressgateway
#           kubectl wait --for=condition=ready -n watch-auditor pod -l app=watch-auditor
#           kubectl wait --for=condition=ready -n pepr-system pod -l app=pepr-soak-ci-watcher

#       - name: Run the soak test and collect metrics
#         run: |
#           # Initialize the map to store pod counts
#           declare -A pod_map
          
#           update_pod_map() {
#             for pod in $(kubectl get pods -n pepr-demo -o jsonpath='{.items[*].metadata.name}'); do
#               count=${pod_map[$pod]}
#               if [ -z "$count" ]; then
#                 pod_map[$pod]=1
#               else
#                 pod_map[$pod]=$((count + 1))
#               fi
#             done
#           }
#           touch logs/auditor-log.txt
#           touch logs/informer-log.txt

#           update_pod_map

#           collect_metrics() {
#             kubectl exec metrics-collector -n watch-auditor -- curl watch-auditor:8080/metrics | grep watch_controller_failures_total > logs/auditor-log.txt
#             kubectl exec metrics-collector -n watch-auditor -- curl -k https://pepr-soak-ci-watcher.pepr-system.svc.cluster.local/metrics | egrep -E "pepr_cache_miss|pepr_resync_failure_count" > logs/informer-log.txt
#           }

#           # Start collecting metrics every 5 minutes and checking pod counts every 30 minutes
#           for i in {1..27}; do  # 27 iterations cover 135 minutes (2 hours and 15 minutes)
#             collect_metrics
#             cat logs/informer-log.txt
#             cat logs/auditor-log.txt
#             if [ $((i % 6)) -eq 0 ]; then  # Every 30 minutes 
#               update_pod_map

#               # Verify that no pod's count exceeds 1
#               for pod in "${!pod_map[@]}"; do
#                 if [ "${pod_map[$pod]}" -gt 1 ]; then
#                   echo "Test failed: Pod $pod has count ${pod_map[$pod]}"
#                   exit 1
#                 fi
#               done
#             fi
#             sleep 300s  # Sleep for 5 minutes before the next iteration
#           done

#           echo "Soak test passed successfully!"
#         shell: bash

#       - name: Upload logs
#         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
#         with:
#           name: soak-test-logs
#           path: logs