ArgoCD - local UI deployment (#1686)

Here we integrate the wbaas UI into ArgoCD in a simpler way without additional templating and applicationsets (in contrast to what was tried in #1662). It uses plain yaml values files generated by helmfile via bin/generate-values.

./bin/generate-values
 $ ./bin/generate-values 
error: missing environment

usage: generate-values <environment> <release-name> [output-file-template]
The script can be run like ./bin/generate-values local ui where local could be staging or production and ui any other helmfile release. The third parameter let's you define a different helmfile than k8s/helmfile/helmfile.yaml. This is needed for the CI script. Once we moved all components to Argo we could create a Makefile target that generates/updates all values files, for our convenience.

.github/workflows/check-generated-values.yml
This is a new GitHub workflow that runs the script to check if the checked-in values files are actually up to date. It does this by iterating over each generated values file that is present in k8s/argocd/ and runs generate-values to compare it against.

How to test this locally
Mind my GitHub comments about the targetRevision in both Application manifests.

Prior discussion
Paired on & discussed with tom

Bug: T360876
Co-authored-by: Thomas Arrow <[email protected]>

.github/workflows/check-generated-values.yml

@@ -0,0 +1,59 @@
+on: push
+name: Check values files
+jobs:
+  diff:
+    runs-on: ubuntu-latest
+    env:
+      TMP_DIR: "/tmp/shared"
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Create directories
+        run: |
+          mkdir -p ~/.local/bin
+          mkdir -p "${TMP_DIR}"
+          chmod 777 "${TMP_DIR}"
+      - name: Create helmfile docker shim
+        run: |
+          docker pull ghcr.io/helmfile/helmfile:latest
+
+          echo 'docker run \
+            --rm \
+            --volume "${TMP_DIR}:${TMP_DIR}" \
+            --volume "${PWD}:/workdir" \
+            --workdir /workdir \
+            --user $(id -u):$(id -g) \
+            ghcr.io/helmfile/helmfile:latest helmfile $*' \
+            | tee ~/.local/bin/helmfile
+
+          chmod +x ~/.local/bin/helmfile
+      - name: Create yq docker shim
+        run: |
+          docker pull mikefarah/yq:latest
+
+          echo 'docker run \
+            --rm \
+            --volume "${TMP_DIR}:${TMP_DIR}" \
+            --volume "${PWD}:/workdir" \
+            --workdir /workdir \
+            --user $(id -u):$(id -g) \
+            mikefarah/yq:latest $*' \
+            | tee ~/.local/bin/yq
+
+          chmod +x ~/.local/bin/yq
+      - name: Diff current values files against generated ones
+        run: >
+          set -x;
+          for ENV_DIR in k8s/argocd/*; do
+            ENV=$(basename "${ENV_DIR}")
+
+            for RELEASE_FILE in "${ENV_DIR}"/*.values.yaml; do
+              RELEASE=$(basename "${RELEASE_FILE}" .values.yaml)
+              TMP_VALUES="${TMP_DIR}"/tmp_"${ENV}.${RELEASE}".yml
+
+              echo "checking $RELEASE_FILE - [$ENV] [$RELEASE]"
+
+              ./bin/generate-values "${ENV}" "${RELEASE}" "${TMP_VALUES}"
+              diff "${TMP_VALUES}" "${RELEASE_FILE}"
+            done
+          done

bin/generate-values

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+function usage() {
+    echo
+    echo "usage: $(basename $0) <environment> <release-name> [output-file-template]"
+    echo
+}
+
+ENVIRONMENT="$1"
+RELEASE="$2"
+
+# absolute path of the wbaas-deploy repository
+ROOT=$(realpath $(dirname $(realpath $BASH_SOURCE))/..)
+OUTPUT_TEMPLATE="${ROOT}/k8s/argocd/${ENVIRONMENT}/${RELEASE}.values.yaml"
+HELMFILE="k8s/helmfile/helmfile.yaml"
+TMP_HELMFILE="$(dirname ${HELMFILE})/.tmp_helmfile.$(mktemp -u XXXXXX).yaml"
+
+if [[ -n "$3" ]]; then
+    OUTPUT_TEMPLATE="$3"
+fi
+
+if [[ ! -e "${HELMFILE}" ]]; then
+    echo "error: helmfile not found: '${HELMFILE}'"
+    usage
+    exit 1 
+fi
+
+if [[ -z "${ENVIRONMENT}" ]]; then
+    echo "error: missing environment"
+    usage
+    exit 2
+fi
+
+if [[ -z "${RELEASE}" ]]; then
+    echo "error: missing release name"
+    usage
+    exit 3
+fi
+
+echo "environment: ${ENVIRONMENT}"
+echo "release: ${RELEASE}"
+
+# modify tmp helmfile by setting each release as "installed", so it always gets processed
+cp "${HELMFILE}" "${TMP_HELMFILE}"
+sed -i 's/installed: .*$/installed: true/g' "${TMP_HELMFILE}"
+
+helmfile \
+    --file "${TMP_HELMFILE}" \
+    --environment "${ENVIRONMENT}" \
+    --selector name="${RELEASE}" \
+    --output-file-template "${OUTPUT_TEMPLATE}" \
+    --skip-deps \
+    write-values
+
+rm "${TMP_HELMFILE}"
+
+# fix indentation in output file for yamllint action
+yq -I 2 -i "${OUTPUT_TEMPLATE}"

charts/argocd-apps/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: argocd-apps
+description: Chart to deploy WBaaS apps in an "app-of-apps" pattern via ArgoCD
+type: application
+version: 0.1.0
+appVersion: "1.0"

charts/argocd-apps/templates/wbaas-ui.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ui
+  namespace: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  destination:
+    namespace: default
+    server: {{ .Values.clusterUrl }}
+  project: {{ .Values.environment }}
+  sources:
+    - repoURL: {{ .Values.repoUrls.charts }}
+      path: charts/ui
+      targetRevision: HEAD
+      helm:
+        valueFiles:
+          - $values/k8s/argocd/{{ .Values.environment }}/ui.values.yaml
+    - repoURL: {{ .Values.repoUrls.deploy }}
+      targetRevision: HEAD
+      ref: values
+
+  syncPolicy:
+    automated:
+      # disable self-healing for local env so we can use skaffold
+      selfHeal: {{- if eq .Values.environment "local" }} false {{ else }} true {{ end}}
+      prune: true

charts/argocd-apps/values.yaml

@@ -0,0 +1,3 @@
+clusterUrl: https://kubernetes.default.svc
+
+# "inherits" values from argocd-config chart

charts/argocd-config/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/argocd-config/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: argocd-apps
+description: Chart to deploy ArgoCD configuration (including the argocd-apps chart)
+type: application
+version: 0.1.0
+appVersion: "1.0"

charts/argocd-config/templates/app-of-apps.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: app-of-apps
+spec:
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  project: {{ .Values.environment }}
+  source:
+    path: charts/argocd-apps
+    repoURL: {{ .Values.repoUrls.deploy }}
+    targetRevision: HEAD
+    helm:
+      values: |
+{{ toYaml .Values | indent 8 }}
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

charts/argocd-config/templates/projects.yaml

@@ -0,0 +1,16 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: {{ .Values.environment }}
+spec:
+  description: The {{ .Values.environment }} deployment of wikibase.cloud
+  destinations:
+  - name: in-cluster-default
+    namespace: default
+    server: https://kubernetes.default.svc
+  - name: in-cluster-argocd
+    namespace: argocd
+    server: https://kubernetes.default.svc
+  sourceRepos:
+  - {{ .Values.repoUrls.deploy }}
+  - {{ .Values.repoUrls.charts }}

charts/argocd-config/values.yaml

@@ -0,0 +1,5 @@
+environment: production
+
+repoUrls:
+  deploy: https://github.com/wmde/wbaas-deploy
+  charts: https://github.com/wbstack/charts.git

doc/deployments/argocd.md

@@ -1,4 +1,49 @@
 # Argo CD
+## Overview
+### Deployment of Argo CD
+We deploy Argo CD via helmfile and the community helm charts: https://argoproj.github.io/argo-helm/
+- see [/k8s/helmfile/argo-cd.yaml](../../k8s/helmfile/argo-cd.yaml)
+
+It's basic configuration lives in the values files `argo-cd-base.values.yaml.gotmpl` for each environment.
+ - [production](../../k8s/helmfile/env/production/argo-cd-base.values.yaml.gotmpl)
+     - [staging](../../k8s/helmfile/env/staging/argo-cd-base.values.yaml.gotmpl)
+    - [local](../../k8s/helmfile/env/local/argo-cd-base.values.yaml.gotmpl)
+
+Currently this means each environment gets it's own instance of Argo CD, which in turn always deploys to the cluster it lives in. We may want to change this in the future, if this prevents us from using certain features or workflows.
+
+### Application configuration
+There are two more charts we use to configure our project & applications:
+- [argocd-config](../../charts/argocd-config/)
+  - gets deployed by helmfile
+  - defines the ["app-of-apps"](https://argo-cd.readthedocs.io/en/stable/operator-manual/cluster-bootstrapping/) `Application` 
+- [argocd-apps](../../charts/argocd-apps/)
+  - gets deployed by Argo CD
+  - defines the `Application` resources for our actual helm releases
+
+The values of the first chart get passed on to the second one, so we can infer which environment we are running in.
+
+### Values files
+After trying out different ways of templating our way through this, we currently settled with a script to generate plain yaml files:
+- the bash script: [/bin/generate/values](../../bin/generate-values)
+- the values files: [/k8s/argocd/](../../k8s/argocd/)
+
+These are generated by helmfile, which in turn reads the value files in [/k8s/helmfile/env/](../../k8s/helmfile/env/) like we are used to.
+
+#### [/bin/generate/values](../../bin/generate-values)
+```
+ $ ./bin/generate-values 
+error: missing environment
+
+usage: generate-values <environment> <release-name> [output-file-template]
+```
+The script can be run like `./bin/generate-values local ui` where `local` could be `staging` or `production` and `ui` any other helmfile release. The third parameter let's you define a different helmfile than `k8s/helmfile/helmfile.yaml`. This is needed for the CI script. Once we moved all components to Argo we could create a Makefile target that generates/updates all values files, for our convenience.
+
+#### CI - [check-generated-values.yml](../../.github/workflows/check-generated-values.yml)
+This GitHub workflow runs the script to check if the checked-in values files are actually up to date. It does this by iterating over each generated values file that is present in `k8s/argocd/` and runs `generate-values` to compare it against.
+
+### Self-Healing
+[Automatic Self-Healing](https://argo-cd.readthedocs.io/en/stable/user-guide/auto_sync/#automatic-self-healing) is currently disabled for the local environment. This way, we still can use skaffold to replace resources that get deployed by Argo CD. Otherwise Argo CD would immediately replace them again with the image that is configured in the values files.
+
 ## Admin access
 > Caution! Admin access should only happen in rare circumstances (testing/diagnosing, for example)
 > as we want to maintain the configuration for everything in git.

k8s/argocd/local/ui.values.yaml

@@ -0,0 +1,29 @@
+image:
+  tag: sha-840d55d
+ingress:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/from-to-www-redirect: "true"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+  enabled: true
+  hosts:
+    - host: www.wbaas.localhost
+      paths:
+        - /*
+  tls: null
+podLabels:
+  sidecar.istio.io/inject: "true"
+resources:
+  limits:
+    cpu: 10m
+    memory: 20Mi
+  requests:
+    cpu: 1m
+    memory: 6Mi
+ui:
+  apiUrl: http://api.wbaas.localhost
+  cnameConfigMapKey: cname_record
+  configMapName: wbaas-ui-config
+  recaptchaSitekeySecretKey: site_key
+  recaptchaSitekeySecretName: recaptcha-v3-secrets
+  subdomainSuffix: .wbaas.localhost

k8s/argocd/production/ui.values.yaml

@@ -0,0 +1,32 @@
+image:
+  tag: sha-840d55d
+ingress:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/from-to-www-redirect: "true"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+  enabled: true
+  hosts:
+    - host: www.wikibase.cloud
+      paths:
+        - /*
+  tls:
+    - hosts:
+        - www.wikibase.cloud
+      secretName: wikibase-production-tls
+podLabels:
+  sidecar.istio.io/inject: "true"
+resources:
+  limits:
+    cpu: 10m
+    memory: 20Mi
+  requests:
+    cpu: 1m
+    memory: 6Mi
+ui:
+  apiUrl: https://api.wikibase.cloud
+  cnameConfigMapKey: cname_record
+  configMapName: wbaas-ui-config
+  recaptchaSitekeySecretKey: site_key
+  recaptchaSitekeySecretName: recaptcha-v3-secrets
+  subdomainSuffix: .wikibase.cloud

k8s/argocd/staging/ui.values.yaml

@@ -0,0 +1,32 @@
+image:
+  tag: sha-840d55d
+ingress:
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/from-to-www-redirect: "true"
+    nginx.ingress.kubernetes.io/use-regex: "true"
+  enabled: true
+  hosts:
+    - host: www.wikibase.dev
+      paths:
+        - /*
+  tls:
+    - hosts:
+        - www.wikibase.dev
+      secretName: wikibase-dev-tls
+podLabels:
+  sidecar.istio.io/inject: "true"
+resources:
+  limits:
+    cpu: 10m
+    memory: 20Mi
+  requests:
+    cpu: 1m
+    memory: 6Mi
+ui:
+  apiUrl: https://api.wikibase.dev
+  cnameConfigMapKey: cname_record
+  configMapName: wbaas-ui-config
+  recaptchaSitekeySecretKey: site_key
+  recaptchaSitekeySecretName: recaptcha-v3-secrets
+  subdomainSuffix: .wikibase.dev

k8s/helmfile/env/local/argocd-config.values.yaml.gotmpl

@@ -0,0 +1,2 @@
+environment: local
+

k8s/helmfile/env/production/argocd-config.values.yaml.gotmpl

@@ -0,0 +1,2 @@
+environment: production
+

k8s/helmfile/env/staging/argocd-config.values.yaml.gotmpl

@@ -0,0 +1,2 @@
+environment: staging
+