From 69ac9e68a0e986b55b09b45bd93e1fbbe5158208 Mon Sep 17 00:00:00 2001
From: Alexei Dodon <adodon@cisco.com>
Date: Mon, 16 Oct 2023 21:00:52 +0300
Subject: [PATCH] ci(nightly): add prometheus kind test

---
 .github/workflows/nightly.yaml                | 134 +-----------------
 examples/kind/kind_ci.sh                      | 106 ++++++++++++++
 .../zot-extended/patch-deployment.yaml        |   8 ++
 3 files changed, 119 insertions(+), 129 deletions(-)
 create mode 100755 examples/kind/kind_ci.sh
 create mode 100644 examples/metrics/kubernetes/zot-extended/patch-deployment.yaml

diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
index 0a0a7ae968..bcda3507cf 100644
--- a/.github/workflows/nightly.yaml
+++ b/.github/workflows/nightly.yaml
@@ -1,7 +1,7 @@
 name: 'Nightly jobs'
 on:
-  schedule:
-    - cron: '30 1 * * *'
+  pull_request:
+    branches: [main]
   workflow_dispatch:
 
 permissions: read-all
@@ -13,7 +13,7 @@ permissions: read-all
 #  task scheduler will start a restore all blobs process at zot startup, after it finishes all blobs should be restored to their original state (have content)
 jobs:
   dedupe:
-    name: Dedupe/restore blobs
+    name: Kind Prometheus test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -24,8 +24,6 @@ jobs:
       - name: Install dependencies
         run: |
           cd $GITHUB_WORKSPACE
-          go install github.com/swaggo/swag/cmd/swag@v1.8.12
-          go mod download
           sudo apt-get update
           sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
           # install skopeo
@@ -40,127 +38,5 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ github.token }}
-      - name: Install localstack
-        run: |
-          pip install --upgrade pyopenssl
-          pip install localstack==2.3.1 awscli-local[ver1] # install LocalStack cli and awslocal
-          docker pull ghcr.io/project-zot/ci-images/localstack:2.3.1 # Make sure to pull the latest version of the image
-          localstack start -d                       # Start LocalStack in the background
-
-          echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
-          localstack wait -t 30                     # to become ready before timing out
-          echo "Startup complete"
-      - name: Run blackbox nightly dedupe tests
-        run: |
-            # test restoring s3 blobs after cache is deleted
-            # test deduping filesystem blobs after switching dedupe to enable
-            make run-blackbox-dedupe-nightly
-        env:
-          AWS_ACCESS_KEY_ID: fake
-          AWS_SECRET_ACCESS_KEY: fake
-      - uses: ./.github/actions/teardown-localstack
-
-  sync:
-    name: Sync harness
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out source code
-        uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.20.x
-      - name: Install dependencies
-        run: |
-          cd $GITHUB_WORKSPACE
-          go install github.com/swaggo/swag/cmd/swag@v1.8.12
-          go mod download
-      - name: Run sync harness
-        run: |
-            make run-blackbox-sync-nightly
-
-  gc-referrers-stress-s3:
-    name: GC(with referrers) on S3(localstack) with short interval
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
-      - uses: actions/setup-go@v4
-        with:
-          cache: false
-          go-version: 1.20.x
-      - uses: ./.github/actions/setup-localstack
-
-      - name: Run zb
-        id: bench
-        run: |
-            make binary
-            make bench
-            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json &
-            sleep 10
-            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup
-
-            killall -r zot-*
-
-            # clean zot storage
-            sudo rm -rf /tmp/zot
-        env:
-          AWS_ACCESS_KEY_ID: fake
-          AWS_SECRET_ACCESS_KEY: fake
-        continue-on-error: true
-
-      - name: Check on failures
-        if: steps.bench.outcome != 'success'
-        run: |
-          cat /tmp/gc-referrers-bench-s3.log
-          exit 1
-      - uses: ./.github/actions/teardown-localstack
-
-  gc-stress-s3:
-    name: GC(without referrers) on S3(localstack) with short interval
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
-      - uses: actions/setup-go@v4
-        with:
-          cache: false
-          go-version: 1.20.x
-      - uses: ./.github/actions/setup-localstack
-
-      - name: Run zb
-        id: bench
-        run: |
-            make binary
-            make bench
-            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json &
-            sleep 10
-            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup
-
-            killall -r zot-*
-
-            # clean zot storage
-            sudo rm -rf /tmp/zot
-        env:
-          AWS_ACCESS_KEY_ID: fake
-          AWS_SECRET_ACCESS_KEY: fake
-        continue-on-error: true
-
-      - name: Check on failures
-        if: steps.bench.outcome != 'success'
-        run: |
-          cat /tmp/gc-bench-s3.log
-          exit 1
-      - uses: ./.github/actions/teardown-localstack
-
-  docker-image:
-    name: Build docker image (for users still using Docker environments)
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
-      - name: Check out source code
-        uses: actions/checkout@v4
-      - name: Build image
-        run: |
-          make docker-image
+      - name: Run tests
+        run: ./examples/kind/kind_ci.sh
diff --git a/examples/kind/kind_ci.sh b/examples/kind/kind_ci.sh
new file mode 100755
index 0000000000..2d43edc217
--- /dev/null
+++ b/examples/kind/kind_ci.sh
@@ -0,0 +1,106 @@
+#!/bin/sh
+set -o errexit
+
+# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
+
+# set no_proxy if applicable
+if [ ! -z "${no_proxy}" ]; then
+  echo "Updating no_proxy env var";
+  export no_proxy=${no_proxy},kind-registry;
+  export NO_PROXY=${no_proxy};
+fi
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    ghcr.io/project-zot/zot-minimal-linux-amd64:latest
+fi
+
+CLUSTER_NAME=kind
+## Delete the cluster if it already exist
+kind get clusters | grep ${CLUSTER_NAME} &&  kind delete cluster --name ${CLUSTER_NAME}
+
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_name}:5000"]
+EOF
+
+# connect the registry to the cluster network if not already connected
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+#
+# document the local registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+
+kind load docker-image quay.io/prometheus-operator/prometheus-operator:v0.51.2 --name ${CLUSTER_NAME}
+kind load docker-image quay.io/prometheus-operator/prometheus-config-reloader:v0.51.2 --name ${CLUSTER_NAME}
+kind load docker-image quay.io/prometheus/prometheus:v2.22.1 --name ${CLUSTER_NAME}
+
+## Deploy prometheus operator
+kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
+
+## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
+kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
+kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
+kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
+
+make oci-image
+# copy the image
+skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:latest
+
+# deploy the image
+kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
+kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
+kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
+kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
+
+# check for availability
+echo "Waiting for deployment/zot-extended to be ready ..."
+kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
+kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
+
+kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
+kubectl get pods -A
+sleep 10
+echo "Kind cluster status:"
+kubectl get pods -A
+
+containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
+echo "containername=${containername}"
+if [ "${containername}" != '"zot-extended"' ]; then
+    exit 1
+fi
+
+containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
+echo "containerup=${containerup}"
+if [ "${containerup}" != '"1"' ]; then
+    exit 1
+fi
+
+zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
+echo "zotinfo=${zotinfo}"
+if [ "${zotinfo}" != '"0"' ]; then
+    exit 1
+fi
diff --git a/examples/metrics/kubernetes/zot-extended/patch-deployment.yaml b/examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
new file mode 100644
index 0000000000..c4e76b9e26
--- /dev/null
+++ b/examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
@@ -0,0 +1,8 @@
+spec:
+  template:
+    spec:
+      containers:
+      - name: zot-extended
+        image: localhost:5001/zot-build:latest
+        command: ["/usr/local/bin/zot-linux-amd64"]
+