fix(chart): job to patch scaledobject stuck in deleting (#2222)

Signed-off-by: Viet Nguyen Duc <[email protected]>
SeleniumHQ · Apr 25, 2024 · a83f0d6 · a83f0d6
1 parent f39a9da
commit a83f0d6
Show file tree

Hide file tree

Showing 16 changed files with 157 additions and 18 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -51,7 +51,11 @@ jobs:
         env:
           AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }}
       - name: Build Docker images
-        run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
+        uses: nick-invision/retry@master
+        with:
+          timeout_minutes: 12
+          max_attempts: 3
+          command: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
       - name: Count image layers
         run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make count_image_layers
       - name: Test Docker images

diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -116,7 +116,11 @@ jobs:
           echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV
           echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV
       - name: Build Docker images
-        run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
+        uses: nick-invision/retry@master
+        with:
+          timeout_minutes: 12
+          max_attempts: 3
+          command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
       - name: Setup Kubernetes cluster
         uses: nick-invision/retry@master
         with:

diff --git a/.github/workflows/test-video.yml b/.github/workflows/test-video.yml
@@ -67,11 +67,15 @@ jobs:
         env:
           AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }}
       - name: Pre-build to reduce logs in test phase
-        run: |
-          VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make hub
-          VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chrome
-          VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make firefox
-          VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make edge
+        uses: nick-invision/retry@master
+        with:
+          timeout_minutes: 12
+          max_attempts: 3
+          command: |
+            VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make hub
+            VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chrome
+            VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make firefox
+            VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make edge
       - name: Set test parameters
         if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
         run: |

diff --git a/Base/Dockerfile b/Base/Dockerfile
@@ -55,7 +55,7 @@ RUN  echo "deb http://archive.ubuntu.com/ubuntu jammy main universe\n" > /etc/ap
     gnupg2 \
     libnss3-tools \
   && mkdir -p /etc/apt/keyrings \
-  && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 843C48A565F8F04B \
+  && apt-key adv --keyserver hkps://keyserver.ubuntu.com:443 --recv-keys 843C48A565F8F04B || apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 843C48A565F8F04B \
   && wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc >dev/null \
   && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list >dev/null \
   && apt-get -qqy update \

diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl
@@ -39,6 +39,13 @@ app.kubernetes.io/component: {{ printf "selenium-grid-%s" .Chart.AppVersion }}
 helm.sh/chart: {{ include "seleniumGrid.chart" . }}
 {{- end -}}
 
+{{/*
+Autoscaling labels
+*/}}
+{{- define "seleniumGrid.autoscalingLabels" -}}
+component.autoscaling: "true"
+{{- end -}}
+
 {{- define "seleniumGrid.component.name" -}}
 {{- $component := index . 0 }}
 {{- $root := index . 1 }}
@@ -181,3 +188,24 @@ Server ConfigMap fullname
 {{- define "seleniumGrid.server.configmap.fullname" -}}
 {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-server-config" $)) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+Patch scaledObjects finalizers job fullname
+*/}}
+{{- define "seleniumGrid.keda.patchObjectsJob.fullname" -}}
+{{- printf "%s-%s" .Release.Name "patch-scaledobjects-finalizers" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Patch scaled objects RoleBinding fullname
+*/}}
+{{- define "seleniumGrid.keda.roleBinding.fullname" -}}
+{{- printf "%s-%s" .Release.Name "patch-keda-rb" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Patch scaled objects Role fullname
+*/}}
+{{- define "seleniumGrid.keda.role.fullname" -}}
+{{- printf "%s-%s" .Release.Name "patch-keda-role" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml
@@ -11,6 +11,7 @@ metadata:
   labels:
     deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }}
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }}
     {{- with .Values.chromeNode.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}

diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml
@@ -12,6 +12,7 @@ metadata:
     app: {{ template "seleniumGrid.chromeNode.fullname" . }}
     app.kubernetes.io/name: {{ template "seleniumGrid.chromeNode.fullname" . }}
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }}
     {{- with .Values.chromeNode.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}

diff --git a/charts/selenium-grid/templates/edge-node-hpa.yaml b/charts/selenium-grid/templates/edge-node-hpa.yaml
@@ -11,6 +11,7 @@ metadata:
   labels:
     deploymentName: {{ template "seleniumGrid.edgeNode.fullname" . }}
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }}
     {{- with .Values.edgeNode.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}

diff --git a/charts/selenium-grid/templates/edge-node-scaledjob.yaml b/charts/selenium-grid/templates/edge-node-scaledjob.yaml
@@ -12,6 +12,7 @@ metadata:
     app: {{ template "seleniumGrid.edgeNode.fullname" . }}
     app.kubernetes.io/name: {{ template "seleniumGrid.edgeNode.fullname" . }}
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }}
     {{- with .Values.edgeNode.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}

diff --git a/charts/selenium-grid/templates/firefox-node-hpa.yaml b/charts/selenium-grid/templates/firefox-node-hpa.yaml
@@ -11,6 +11,7 @@ metadata:
   labels:
     deploymentName: {{ template "seleniumGrid.firefoxNode.fullname" . }}
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }}
     {{- with .Values.firefoxNode.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}

diff --git a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml
@@ -12,6 +12,7 @@ metadata:
     app: {{ template "seleniumGrid.firefoxNode.fullname" . }}
     app.kubernetes.io/name: {{ template "seleniumGrid.firefoxNode.fullname" . }}
     {{- include "seleniumGrid.commonLabels" . | nindent 4 }}
+    {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }}
     {{- with .Values.firefoxNode.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}

diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml
@@ -0,0 +1,35 @@
+{{- if eq (include "seleniumGrid.useKEDA" $) "true" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook-delete-policy": hook-succeeded
+    "helm.sh/resource-policy": delete
+    {{- with $.Values.autoscaling.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
+    {{- with $.Values.customLabels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  template:
+    metadata:
+      name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+    spec:
+      activeDeadlineSeconds: 120
+      serviceAccountName: {{ template "seleniumGrid.serviceAccount.fullname" $ }}
+      serviceAccount: {{ template "seleniumGrid.serviceAccount.fullname" $ }}
+      containers:
+        - name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+          image: {{ $.Values.global.seleniumGrid.kubectlImage }}
+          command:
+            - "bin/bash"
+            - "-c"
+            - "kubectl get ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling=true -o=json | jq '.metadata.finalizers = null' | kubectl apply -f -"
+      restartPolicy: Never
+{{- end }}
diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-rb.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-rb.yaml
@@ -0,0 +1,20 @@
+{{- if eq (include "seleniumGrid.useKEDA" $) "true" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "seleniumGrid.keda.roleBinding.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.keda.roleBinding.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
+    {{- with $.Values.customLabels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "seleniumGrid.serviceAccount.fullname" $ }}
+roleRef:
+  kind: Role
+  name: {{ template "seleniumGrid.keda.role.fullname" $ }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-role.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-role.yaml
@@ -0,0 +1,37 @@
+{{- if eq (include "seleniumGrid.useKEDA" $) "true" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "seleniumGrid.keda.role.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.keda.role.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
+    {{- with $.Values.customLabels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+rules:
+  - apiGroups:
+      - keda.sh
+    resources:
+      - scaledjobs
+      - scaledjobs/finalizers
+      - scaledjobs/status
+    verbs:
+      - '*'
+  - apiGroups:
+      - keda.sh
+    resources:
+      - scaledobjects
+      - scaledobjects/finalizers
+      - scaledobjects/status
+    verbs:
+      - '*'
+  - apiGroups:
+      - keda.sh
+    resources:
+      - triggerauthentications
+      - triggerauthentications/status
+    verbs:
+      - '*'
+{{- end }}
diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -12,6 +12,8 @@ global:
     nodesImageTag: 4.19.1-20240402
     # Image tag for browser's video recorder
     videoImageTag: ffmpeg-7.0-20240402
+    # kubectl image is used to execute kubectl commands in utility jobs
+    kubectlImage: bitnami/kubectl:latest
     # Pull secret for all components, can be overridden individually
     imagePullSecret: ""
     # Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging
@@ -546,7 +548,7 @@ autoscaling:
   scalingType: job
   # Annotations for KEDA resources: ScaledObject and ScaledJob
   annotations:
-    "helm.sh/hook": post-install
+    "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
   # Options for KEDA scaled resources (keep only common options used for both ScaledJob and ScaledObject)
   scaledOptions:
     minReplicaCount: 0
@@ -1185,7 +1187,7 @@ customLabels: {}
 # Configuration for dependency chart keda
 keda:
   additionalAnnotations:
-    "helm.sh/hook": pre-install
+    "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete
   http:
     timeout: 60000
   webhooks:
@@ -1221,19 +1223,19 @@ prometheus-stack:
   defaultRules:
     create: true
     annotations:
-      "helm.sh/hook": post-install
+      "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
   alertmanager:
     enabled: true
     annotations:
-      "helm.sh/hook": post-install
+      "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
   grafana:
     enabled: true
     adminPassword: admin
     forceDeployDatasources: true
     forceDeployDashboards: true
     annotations:
-      "helm.sh/hook": post-install
+      "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
   prometheus:
     enabled: true
     annotations:
-      "helm.sh/hook": post-install
+      "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh
@@ -97,19 +97,19 @@ HELM_COMMAND_SET_IMAGES=" \
 --set global.seleniumGrid.logLevel=${LOG_LEVEL} \
 "
 
-if [ "${TEST_EXISTING_KEDA}" = "true" ]; then
+if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${TEST_EXISTING_KEDA}" = "true" ]; then
   HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
   --set autoscaling.enabled=false \
   --set autoscaling.enableWithExistingKEDA=true \
   "
-else
+elif [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${TEST_EXISTING_KEDA}" = "true" ]; then
   HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
   --set autoscaling.enabled=true \
   --set autoscaling.enableWithExistingKEDA=false \
   "
 fi
 
-if [ -n "${SET_MAX_REPLICAS}" ]; then
+if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ -n "${SET_MAX_REPLICAS}" ]; then
   HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \
   --set autoscaling.scaledOptions.maxReplicaCount=${SET_MAX_REPLICAS} \
   "
@@ -151,7 +151,6 @@ fi
 
 if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then
   HELM_COMMAND_SET_AUTOSCALING=" \
-  --set autoscaling.enableWithExistingKEDA=${SELENIUM_GRID_AUTOSCALING} \
   --set autoscaling.scaledOptions.minReplicaCount=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA} \
   "
 fi