From a83f0d6ab9d9337835ced36ceda4a44554d8a12c Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Thu, 25 Apr 2024 01:02:38 +0000 Subject: [PATCH] fix(chart): job to patch scaledobject stuck in deleting (#2222) Signed-off-by: Viet Nguyen Duc --- .github/workflows/build-test.yml | 6 ++- .github/workflows/helm-chart-test.yml | 6 ++- .github/workflows/test-video.yml | 14 ++++--- Base/Dockerfile | 2 +- .../selenium-grid/templates/_nameHelpers.tpl | 28 ++++++++++++++ .../templates/chrome-node-hpa.yaml | 1 + .../templates/chrome-node-scaledjobs.yaml | 1 + .../templates/edge-node-hpa.yaml | 1 + .../templates/edge-node-scaledjob.yaml | 1 + .../templates/firefox-node-hpa.yaml | 1 + .../templates/firefox-node-scaledjob.yaml | 1 + .../patch-keda/patch-keda-objects-job.yaml | 35 ++++++++++++++++++ .../templates/patch-keda/patch-keda-rb.yaml | 20 ++++++++++ .../templates/patch-keda/patch-keda-role.yaml | 37 +++++++++++++++++++ charts/selenium-grid/values.yaml | 14 ++++--- tests/charts/make/chart_test.sh | 7 ++-- 16 files changed, 157 insertions(+), 18 deletions(-) create mode 100644 charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml create mode 100644 charts/selenium-grid/templates/patch-keda/patch-keda-rb.yaml create mode 100644 charts/selenium-grid/templates/patch-keda/patch-keda-role.yaml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ed35ce474..d2a3a5f9c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -51,7 +51,11 @@ jobs: env: AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }} - name: Build Docker images - run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build + uses: nick-invision/retry@master + with: + timeout_minutes: 12 + max_attempts: 3 + command: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - name: Count image layers run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make count_image_layers - name: Test Docker images diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index f4a4a884b..e22eb4831 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -116,7 +116,11 @@ jobs: echo "CHART_PACKAGE_PATH=$(cat /tmp/selenium_chart_version)" >> $GITHUB_ENV echo "CHART_FILE_NAME=$(basename $(cat /tmp/selenium_chart_version))" >> $GITHUB_ENV - name: Build Docker images - run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build + uses: nick-invision/retry@master + with: + timeout_minutes: 12 + max_attempts: 3 + command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - name: Setup Kubernetes cluster uses: nick-invision/retry@master with: diff --git a/.github/workflows/test-video.yml b/.github/workflows/test-video.yml index 202758120..c50025117 100644 --- a/.github/workflows/test-video.yml +++ b/.github/workflows/test-video.yml @@ -67,11 +67,15 @@ jobs: env: AUTHORS: ${{ vars.AUTHORS || 'SeleniumHQ' }} - name: Pre-build to reduce logs in test phase - run: | - VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make hub - VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chrome - VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make firefox - VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make edge + uses: nick-invision/retry@master + with: + timeout_minutes: 12 + max_attempts: 3 + command: | + VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make hub + VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chrome + VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make firefox + VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make edge - name: Set test parameters if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') run: | diff --git a/Base/Dockerfile b/Base/Dockerfile index b3d9fee58..d0be2445d 100644 --- a/Base/Dockerfile +++ b/Base/Dockerfile @@ -55,7 +55,7 @@ RUN echo "deb http://archive.ubuntu.com/ubuntu jammy main universe\n" > /etc/ap gnupg2 \ libnss3-tools \ && mkdir -p /etc/apt/keyrings \ - && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 843C48A565F8F04B \ + && apt-key adv --keyserver hkps://keyserver.ubuntu.com:443 --recv-keys 843C48A565F8F04B || apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 843C48A565F8F04B \ && wget -qO - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc >dev/null \ && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list >dev/null \ && apt-get -qqy update \ diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index e5370aeed..75bc1a847 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -39,6 +39,13 @@ app.kubernetes.io/component: {{ printf "selenium-grid-%s" .Chart.AppVersion }} helm.sh/chart: {{ include "seleniumGrid.chart" . }} {{- end -}} +{{/* +Autoscaling labels +*/}} +{{- define "seleniumGrid.autoscalingLabels" -}} +component.autoscaling: "true" +{{- end -}} + {{- define "seleniumGrid.component.name" -}} {{- $component := index . 0 }} {{- $root := index . 1 }} @@ -181,3 +188,24 @@ Server ConfigMap fullname {{- define "seleniumGrid.server.configmap.fullname" -}} {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-server-config" $)) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Patch scaledObjects finalizers job fullname +*/}} +{{- define "seleniumGrid.keda.patchObjectsJob.fullname" -}} +{{- printf "%s-%s" .Release.Name "patch-scaledobjects-finalizers" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Patch scaled objects RoleBinding fullname +*/}} +{{- define "seleniumGrid.keda.roleBinding.fullname" -}} +{{- printf "%s-%s" .Release.Name "patch-keda-rb" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Patch scaled objects Role fullname +*/}} +{{- define "seleniumGrid.keda.role.fullname" -}} +{{- printf "%s-%s" .Release.Name "patch-keda-role" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml index 8690e0a8b..8dbf85ec4 100644 --- a/charts/selenium-grid/templates/chrome-node-hpa.yaml +++ b/charts/selenium-grid/templates/chrome-node-hpa.yaml @@ -11,6 +11,7 @@ metadata: labels: deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }} {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} {{- with .Values.chromeNode.labels }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml index 5654cd5a0..c858908ce 100644 --- a/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml +++ b/charts/selenium-grid/templates/chrome-node-scaledjobs.yaml @@ -12,6 +12,7 @@ metadata: app: {{ template "seleniumGrid.chromeNode.fullname" . }} app.kubernetes.io/name: {{ template "seleniumGrid.chromeNode.fullname" . }} {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} {{- with .Values.chromeNode.labels }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-hpa.yaml b/charts/selenium-grid/templates/edge-node-hpa.yaml index c937bc9fb..513440d63 100644 --- a/charts/selenium-grid/templates/edge-node-hpa.yaml +++ b/charts/selenium-grid/templates/edge-node-hpa.yaml @@ -11,6 +11,7 @@ metadata: labels: deploymentName: {{ template "seleniumGrid.edgeNode.fullname" . }} {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} {{- with .Values.edgeNode.labels }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/edge-node-scaledjob.yaml b/charts/selenium-grid/templates/edge-node-scaledjob.yaml index 4a0e4fc51..e9cce337a 100644 --- a/charts/selenium-grid/templates/edge-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/edge-node-scaledjob.yaml @@ -12,6 +12,7 @@ metadata: app: {{ template "seleniumGrid.edgeNode.fullname" . }} app.kubernetes.io/name: {{ template "seleniumGrid.edgeNode.fullname" . }} {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} {{- with .Values.edgeNode.labels }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-hpa.yaml b/charts/selenium-grid/templates/firefox-node-hpa.yaml index 4ed715e67..0d4d907ef 100644 --- a/charts/selenium-grid/templates/firefox-node-hpa.yaml +++ b/charts/selenium-grid/templates/firefox-node-hpa.yaml @@ -11,6 +11,7 @@ metadata: labels: deploymentName: {{ template "seleniumGrid.firefoxNode.fullname" . }} {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} {{- with .Values.firefoxNode.labels }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml index 732ec8c9b..d65bf0b69 100644 --- a/charts/selenium-grid/templates/firefox-node-scaledjob.yaml +++ b/charts/selenium-grid/templates/firefox-node-scaledjob.yaml @@ -12,6 +12,7 @@ metadata: app: {{ template "seleniumGrid.firefoxNode.fullname" . }} app.kubernetes.io/name: {{ template "seleniumGrid.firefoxNode.fullname" . }} {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- include "seleniumGrid.autoscalingLabels" . | nindent 4 }} {{- with .Values.firefoxNode.labels }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml new file mode 100644 index 000000000..d1c21a96f --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml @@ -0,0 +1,35 @@ +{{- if eq (include "seleniumGrid.useKEDA" $) "true" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook-delete-policy": hook-succeeded + "helm.sh/resource-policy": delete + {{- with $.Values.autoscaling.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + deploymentName: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + template: + metadata: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + spec: + activeDeadlineSeconds: 120 + serviceAccountName: {{ template "seleniumGrid.serviceAccount.fullname" $ }} + serviceAccount: {{ template "seleniumGrid.serviceAccount.fullname" $ }} + containers: + - name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + image: {{ $.Values.global.seleniumGrid.kubectlImage }} + command: + - "bin/bash" + - "-c" + - "kubectl get ScaledObjects,ScaledJobs -n {{ .Release.Namespace }} -l component.autoscaling=true -o=json | jq '.metadata.finalizers = null' | kubectl apply -f -" + restartPolicy: Never +{{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-rb.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-rb.yaml new file mode 100644 index 000000000..6e80973b4 --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-rb.yaml @@ -0,0 +1,20 @@ +{{- if eq (include "seleniumGrid.useKEDA" $) "true" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "seleniumGrid.keda.roleBinding.fullname" $ }} + namespace: {{ .Release.Namespace }} + labels: + deploymentName: {{ template "seleniumGrid.keda.roleBinding.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "seleniumGrid.serviceAccount.fullname" $ }} +roleRef: + kind: Role + name: {{ template "seleniumGrid.keda.role.fullname" $ }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-role.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-role.yaml new file mode 100644 index 000000000..6a841421a --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-role.yaml @@ -0,0 +1,37 @@ +{{- if eq (include "seleniumGrid.useKEDA" $) "true" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "seleniumGrid.keda.role.fullname" $ }} + namespace: {{ .Release.Namespace }} + labels: + deploymentName: {{ template "seleniumGrid.keda.role.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - keda.sh + resources: + - scaledjobs + - scaledjobs/finalizers + - scaledjobs/status + verbs: + - '*' + - apiGroups: + - keda.sh + resources: + - scaledobjects + - scaledobjects/finalizers + - scaledobjects/status + verbs: + - '*' + - apiGroups: + - keda.sh + resources: + - triggerauthentications + - triggerauthentications/status + verbs: + - '*' +{{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index a6feec21a..646f69e28 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -12,6 +12,8 @@ global: nodesImageTag: 4.19.1-20240402 # Image tag for browser's video recorder videoImageTag: ffmpeg-7.0-20240402 + # kubectl image is used to execute kubectl commands in utility jobs + kubectlImage: bitnami/kubectl:latest # Pull secret for all components, can be overridden individually imagePullSecret: "" # Log level for all components. Possible values describe here: https://www.selenium.dev/documentation/grid/configuration/cli_options/#logging @@ -546,7 +548,7 @@ autoscaling: scalingType: job # Annotations for KEDA resources: ScaledObject and ScaledJob annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete # Options for KEDA scaled resources (keep only common options used for both ScaledJob and ScaledObject) scaledOptions: minReplicaCount: 0 @@ -1185,7 +1187,7 @@ customLabels: {} # Configuration for dependency chart keda keda: additionalAnnotations: - "helm.sh/hook": pre-install + "helm.sh/hook": pre-install,pre-upgrade,pre-rollback,post-delete http: timeout: 60000 webhooks: @@ -1221,19 +1223,19 @@ prometheus-stack: defaultRules: create: true annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete alertmanager: enabled: true annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete grafana: enabled: true adminPassword: admin forceDeployDatasources: true forceDeployDashboards: true annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete prometheus: enabled: true annotations: - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 152f41cc1..c9e9be8e4 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -97,19 +97,19 @@ HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.logLevel=${LOG_LEVEL} \ " -if [ "${TEST_EXISTING_KEDA}" = "true" ]; then +if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${TEST_EXISTING_KEDA}" = "true" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set autoscaling.enabled=false \ --set autoscaling.enableWithExistingKEDA=true \ " -else +elif [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ "${TEST_EXISTING_KEDA}" = "true" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set autoscaling.enabled=true \ --set autoscaling.enableWithExistingKEDA=false \ " fi -if [ -n "${SET_MAX_REPLICAS}" ]; then +if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ] && [ -n "${SET_MAX_REPLICAS}" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set autoscaling.scaledOptions.maxReplicaCount=${SET_MAX_REPLICAS} \ " @@ -151,7 +151,6 @@ fi if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then HELM_COMMAND_SET_AUTOSCALING=" \ - --set autoscaling.enableWithExistingKEDA=${SELENIUM_GRID_AUTOSCALING} \ --set autoscaling.scaledOptions.minReplicaCount=${SELENIUM_GRID_AUTOSCALING_MIN_REPLICA} \ " fi