From 31f4e622e22506106fd8b99fefebd0f87e15eaed Mon Sep 17 00:00:00 2001 From: Deniz Erdogan <91744937+deer-wmde@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:49:25 +0200 Subject: [PATCH] ArgoCD - local UI deployment (#1686) Here we integrate the wbaas UI into ArgoCD in a simpler way without additional templating and applicationsets (in contrast to what was tried in #1662). It uses plain yaml values files generated by helmfile via bin/generate-values. ./bin/generate-values $ ./bin/generate-values error: missing environment usage: generate-values [output-file-template] The script can be run like ./bin/generate-values local ui where local could be staging or production and ui any other helmfile release. The third parameter let's you define a different helmfile than k8s/helmfile/helmfile.yaml. This is needed for the CI script. Once we moved all components to Argo we could create a Makefile target that generates/updates all values files, for our convenience. .github/workflows/check-generated-values.yml This is a new GitHub workflow that runs the script to check if the checked-in values files are actually up to date. It does this by iterating over each generated values file that is present in k8s/argocd/ and runs generate-values to compare it against. How to test this locally Mind my GitHub comments about the targetRevision in both Application manifests. Prior discussion Paired on & discussed with tom Bug: T360876 Co-authored-by: Thomas Arrow --- .github/workflows/check-generated-values.yml | 59 +++++++++++++++++++ bin/generate-values | 58 ++++++++++++++++++ charts/argocd-apps/Chart.yaml | 6 ++ charts/argocd-apps/templates/wbaas-ui.yaml | 28 +++++++++ charts/argocd-apps/values.yaml | 3 + charts/argocd-config/.helmignore | 23 ++++++++ charts/argocd-config/Chart.yaml | 6 ++ .../argocd-config/templates/app-of-apps.yaml | 20 +++++++ charts/argocd-config/templates/projects.yaml | 16 +++++ charts/argocd-config/values.yaml | 5 ++ doc/deployments/argocd.md | 45 ++++++++++++++ k8s/argocd/local/ui.values.yaml | 29 +++++++++ k8s/argocd/production/ui.values.yaml | 32 ++++++++++ k8s/argocd/staging/ui.values.yaml | 32 ++++++++++ .../local/argocd-config.values.yaml.gotmpl | 2 + .../argocd-config.values.yaml.gotmpl | 2 + .../staging/argocd-config.values.yaml.gotmpl | 2 + k8s/helmfile/helmfile.yaml | 7 +++ 18 files changed, 375 insertions(+) create mode 100644 .github/workflows/check-generated-values.yml create mode 100755 bin/generate-values create mode 100644 charts/argocd-apps/Chart.yaml create mode 100644 charts/argocd-apps/templates/wbaas-ui.yaml create mode 100644 charts/argocd-apps/values.yaml create mode 100644 charts/argocd-config/.helmignore create mode 100644 charts/argocd-config/Chart.yaml create mode 100644 charts/argocd-config/templates/app-of-apps.yaml create mode 100644 charts/argocd-config/templates/projects.yaml create mode 100644 charts/argocd-config/values.yaml create mode 100644 k8s/argocd/local/ui.values.yaml create mode 100644 k8s/argocd/production/ui.values.yaml create mode 100644 k8s/argocd/staging/ui.values.yaml create mode 100644 k8s/helmfile/env/local/argocd-config.values.yaml.gotmpl create mode 100644 k8s/helmfile/env/production/argocd-config.values.yaml.gotmpl create mode 100644 k8s/helmfile/env/staging/argocd-config.values.yaml.gotmpl diff --git a/.github/workflows/check-generated-values.yml b/.github/workflows/check-generated-values.yml new file mode 100644 index 000000000..d260b3b0f --- /dev/null +++ b/.github/workflows/check-generated-values.yml @@ -0,0 +1,59 @@ +on: push +name: Check values files +jobs: + diff: + runs-on: ubuntu-latest + env: + TMP_DIR: "/tmp/shared" + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Create directories + run: | + mkdir -p ~/.local/bin + mkdir -p "${TMP_DIR}" + chmod 777 "${TMP_DIR}" + - name: Create helmfile docker shim + run: | + docker pull ghcr.io/helmfile/helmfile:latest + + echo 'docker run \ + --rm \ + --volume "${TMP_DIR}:${TMP_DIR}" \ + --volume "${PWD}:/workdir" \ + --workdir /workdir \ + --user $(id -u):$(id -g) \ + ghcr.io/helmfile/helmfile:latest helmfile $*' \ + | tee ~/.local/bin/helmfile + + chmod +x ~/.local/bin/helmfile + - name: Create yq docker shim + run: | + docker pull mikefarah/yq:latest + + echo 'docker run \ + --rm \ + --volume "${TMP_DIR}:${TMP_DIR}" \ + --volume "${PWD}:/workdir" \ + --workdir /workdir \ + --user $(id -u):$(id -g) \ + mikefarah/yq:latest $*' \ + | tee ~/.local/bin/yq + + chmod +x ~/.local/bin/yq + - name: Diff current values files against generated ones + run: > + set -x; + for ENV_DIR in k8s/argocd/*; do + ENV=$(basename "${ENV_DIR}") + + for RELEASE_FILE in "${ENV_DIR}"/*.values.yaml; do + RELEASE=$(basename "${RELEASE_FILE}" .values.yaml) + TMP_VALUES="${TMP_DIR}"/tmp_"${ENV}.${RELEASE}".yml + + echo "checking $RELEASE_FILE - [$ENV] [$RELEASE]" + + ./bin/generate-values "${ENV}" "${RELEASE}" "${TMP_VALUES}" + diff "${TMP_VALUES}" "${RELEASE_FILE}" + done + done diff --git a/bin/generate-values b/bin/generate-values new file mode 100755 index 000000000..3683b4f33 --- /dev/null +++ b/bin/generate-values @@ -0,0 +1,58 @@ +#!/bin/bash + +function usage() { + echo + echo "usage: $(basename $0) [output-file-template]" + echo +} + +ENVIRONMENT="$1" +RELEASE="$2" + +# absolute path of the wbaas-deploy repository +ROOT=$(realpath $(dirname $(realpath $BASH_SOURCE))/..) +OUTPUT_TEMPLATE="${ROOT}/k8s/argocd/${ENVIRONMENT}/${RELEASE}.values.yaml" +HELMFILE="k8s/helmfile/helmfile.yaml" +TMP_HELMFILE="$(dirname ${HELMFILE})/.tmp_helmfile.$(mktemp -u XXXXXX).yaml" + +if [[ -n "$3" ]]; then + OUTPUT_TEMPLATE="$3" +fi + +if [[ ! -e "${HELMFILE}" ]]; then + echo "error: helmfile not found: '${HELMFILE}'" + usage + exit 1 +fi + +if [[ -z "${ENVIRONMENT}" ]]; then + echo "error: missing environment" + usage + exit 2 +fi + +if [[ -z "${RELEASE}" ]]; then + echo "error: missing release name" + usage + exit 3 +fi + +echo "environment: ${ENVIRONMENT}" +echo "release: ${RELEASE}" + +# modify tmp helmfile by setting each release as "installed", so it always gets processed +cp "${HELMFILE}" "${TMP_HELMFILE}" +sed -i 's/installed: .*$/installed: true/g' "${TMP_HELMFILE}" + +helmfile \ + --file "${TMP_HELMFILE}" \ + --environment "${ENVIRONMENT}" \ + --selector name="${RELEASE}" \ + --output-file-template "${OUTPUT_TEMPLATE}" \ + --skip-deps \ + write-values + +rm "${TMP_HELMFILE}" + +# fix indentation in output file for yamllint action +yq -I 2 -i "${OUTPUT_TEMPLATE}" diff --git a/charts/argocd-apps/Chart.yaml b/charts/argocd-apps/Chart.yaml new file mode 100644 index 000000000..f44ab6518 --- /dev/null +++ b/charts/argocd-apps/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: argocd-apps +description: Chart to deploy WBaaS apps in an "app-of-apps" pattern via ArgoCD +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/charts/argocd-apps/templates/wbaas-ui.yaml b/charts/argocd-apps/templates/wbaas-ui.yaml new file mode 100644 index 000000000..acc5c3009 --- /dev/null +++ b/charts/argocd-apps/templates/wbaas-ui.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ui + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: default + server: {{ .Values.clusterUrl }} + project: {{ .Values.environment }} + sources: + - repoURL: {{ .Values.repoUrls.charts }} + path: charts/ui + targetRevision: HEAD + helm: + valueFiles: + - $values/k8s/argocd/{{ .Values.environment }}/ui.values.yaml + - repoURL: {{ .Values.repoUrls.deploy }} + targetRevision: HEAD + ref: values + + syncPolicy: + automated: + # disable self-healing for local env so we can use skaffold + selfHeal: {{- if eq .Values.environment "local" }} false {{ else }} true {{ end}} + prune: true diff --git a/charts/argocd-apps/values.yaml b/charts/argocd-apps/values.yaml new file mode 100644 index 000000000..be08010a1 --- /dev/null +++ b/charts/argocd-apps/values.yaml @@ -0,0 +1,3 @@ +clusterUrl: https://kubernetes.default.svc + +# "inherits" values from argocd-config chart diff --git a/charts/argocd-config/.helmignore b/charts/argocd-config/.helmignore new file mode 100644 index 000000000..691fa13d6 --- /dev/null +++ b/charts/argocd-config/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/charts/argocd-config/Chart.yaml b/charts/argocd-config/Chart.yaml new file mode 100644 index 000000000..0fd62e963 --- /dev/null +++ b/charts/argocd-config/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: argocd-apps +description: Chart to deploy ArgoCD configuration (including the argocd-apps chart) +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/charts/argocd-config/templates/app-of-apps.yaml b/charts/argocd-config/templates/app-of-apps.yaml new file mode 100644 index 000000000..bd3fdd5f1 --- /dev/null +++ b/charts/argocd-config/templates/app-of-apps.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: app-of-apps +spec: + destination: + server: https://kubernetes.default.svc + namespace: argocd + project: {{ .Values.environment }} + source: + path: charts/argocd-apps + repoURL: {{ .Values.repoUrls.deploy }} + targetRevision: HEAD + helm: + values: | +{{ toYaml .Values | indent 8 }} + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/charts/argocd-config/templates/projects.yaml b/charts/argocd-config/templates/projects.yaml new file mode 100644 index 000000000..c383dc950 --- /dev/null +++ b/charts/argocd-config/templates/projects.yaml @@ -0,0 +1,16 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: {{ .Values.environment }} +spec: + description: The {{ .Values.environment }} deployment of wikibase.cloud + destinations: + - name: in-cluster-default + namespace: default + server: https://kubernetes.default.svc + - name: in-cluster-argocd + namespace: argocd + server: https://kubernetes.default.svc + sourceRepos: + - {{ .Values.repoUrls.deploy }} + - {{ .Values.repoUrls.charts }} \ No newline at end of file diff --git a/charts/argocd-config/values.yaml b/charts/argocd-config/values.yaml new file mode 100644 index 000000000..828a00c62 --- /dev/null +++ b/charts/argocd-config/values.yaml @@ -0,0 +1,5 @@ +environment: production + +repoUrls: + deploy: https://github.com/wmde/wbaas-deploy + charts: https://github.com/wbstack/charts.git diff --git a/doc/deployments/argocd.md b/doc/deployments/argocd.md index bc6d1878a..e336a3eaa 100755 --- a/doc/deployments/argocd.md +++ b/doc/deployments/argocd.md @@ -1,4 +1,49 @@ # Argo CD +## Overview +### Deployment of Argo CD +We deploy Argo CD via helmfile and the community helm charts: https://argoproj.github.io/argo-helm/ +- see [/k8s/helmfile/argo-cd.yaml](../../k8s/helmfile/argo-cd.yaml) + +It's basic configuration lives in the values files `argo-cd-base.values.yaml.gotmpl` for each environment. + - [production](../../k8s/helmfile/env/production/argo-cd-base.values.yaml.gotmpl) + - [staging](../../k8s/helmfile/env/staging/argo-cd-base.values.yaml.gotmpl) + - [local](../../k8s/helmfile/env/local/argo-cd-base.values.yaml.gotmpl) + +Currently this means each environment gets it's own instance of Argo CD, which in turn always deploys to the cluster it lives in. We may want to change this in the future, if this prevents us from using certain features or workflows. + +### Application configuration +There are two more charts we use to configure our project & applications: +- [argocd-config](../../charts/argocd-config/) + - gets deployed by helmfile + - defines the ["app-of-apps"](https://argo-cd.readthedocs.io/en/stable/operator-manual/cluster-bootstrapping/) `Application` +- [argocd-apps](../../charts/argocd-apps/) + - gets deployed by Argo CD + - defines the `Application` resources for our actual helm releases + +The values of the first chart get passed on to the second one, so we can infer which environment we are running in. + +### Values files +After trying out different ways of templating our way through this, we currently settled with a script to generate plain yaml files: +- the bash script: [/bin/generate/values](../../bin/generate-values) +- the values files: [/k8s/argocd/](../../k8s/argocd/) + +These are generated by helmfile, which in turn reads the value files in [/k8s/helmfile/env/](../../k8s/helmfile/env/) like we are used to. + +#### [/bin/generate/values](../../bin/generate-values) +``` + $ ./bin/generate-values +error: missing environment + +usage: generate-values [output-file-template] +``` +The script can be run like `./bin/generate-values local ui` where `local` could be `staging` or `production` and `ui` any other helmfile release. The third parameter let's you define a different helmfile than `k8s/helmfile/helmfile.yaml`. This is needed for the CI script. Once we moved all components to Argo we could create a Makefile target that generates/updates all values files, for our convenience. + +#### CI - [check-generated-values.yml](../../.github/workflows/check-generated-values.yml) +This GitHub workflow runs the script to check if the checked-in values files are actually up to date. It does this by iterating over each generated values file that is present in `k8s/argocd/` and runs `generate-values` to compare it against. + +### Self-Healing +[Automatic Self-Healing](https://argo-cd.readthedocs.io/en/stable/user-guide/auto_sync/#automatic-self-healing) is currently disabled for the local environment. This way, we still can use skaffold to replace resources that get deployed by Argo CD. Otherwise Argo CD would immediately replace them again with the image that is configured in the values files. + ## Admin access > Caution! Admin access should only happen in rare circumstances (testing/diagnosing, for example) > as we want to maintain the configuration for everything in git. diff --git a/k8s/argocd/local/ui.values.yaml b/k8s/argocd/local/ui.values.yaml new file mode 100644 index 000000000..7b27aba0a --- /dev/null +++ b/k8s/argocd/local/ui.values.yaml @@ -0,0 +1,29 @@ +image: + tag: sha-840d55d +ingress: + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/from-to-www-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" + enabled: true + hosts: + - host: www.wbaas.localhost + paths: + - /* + tls: null +podLabels: + sidecar.istio.io/inject: "true" +resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 1m + memory: 6Mi +ui: + apiUrl: http://api.wbaas.localhost + cnameConfigMapKey: cname_record + configMapName: wbaas-ui-config + recaptchaSitekeySecretKey: site_key + recaptchaSitekeySecretName: recaptcha-v3-secrets + subdomainSuffix: .wbaas.localhost diff --git a/k8s/argocd/production/ui.values.yaml b/k8s/argocd/production/ui.values.yaml new file mode 100644 index 000000000..957808fc1 --- /dev/null +++ b/k8s/argocd/production/ui.values.yaml @@ -0,0 +1,32 @@ +image: + tag: sha-840d55d +ingress: + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/from-to-www-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" + enabled: true + hosts: + - host: www.wikibase.cloud + paths: + - /* + tls: + - hosts: + - www.wikibase.cloud + secretName: wikibase-production-tls +podLabels: + sidecar.istio.io/inject: "true" +resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 1m + memory: 6Mi +ui: + apiUrl: https://api.wikibase.cloud + cnameConfigMapKey: cname_record + configMapName: wbaas-ui-config + recaptchaSitekeySecretKey: site_key + recaptchaSitekeySecretName: recaptcha-v3-secrets + subdomainSuffix: .wikibase.cloud diff --git a/k8s/argocd/staging/ui.values.yaml b/k8s/argocd/staging/ui.values.yaml new file mode 100644 index 000000000..14ad7b137 --- /dev/null +++ b/k8s/argocd/staging/ui.values.yaml @@ -0,0 +1,32 @@ +image: + tag: sha-840d55d +ingress: + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/from-to-www-redirect: "true" + nginx.ingress.kubernetes.io/use-regex: "true" + enabled: true + hosts: + - host: www.wikibase.dev + paths: + - /* + tls: + - hosts: + - www.wikibase.dev + secretName: wikibase-dev-tls +podLabels: + sidecar.istio.io/inject: "true" +resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 1m + memory: 6Mi +ui: + apiUrl: https://api.wikibase.dev + cnameConfigMapKey: cname_record + configMapName: wbaas-ui-config + recaptchaSitekeySecretKey: site_key + recaptchaSitekeySecretName: recaptcha-v3-secrets + subdomainSuffix: .wikibase.dev diff --git a/k8s/helmfile/env/local/argocd-config.values.yaml.gotmpl b/k8s/helmfile/env/local/argocd-config.values.yaml.gotmpl new file mode 100644 index 000000000..77513ccac --- /dev/null +++ b/k8s/helmfile/env/local/argocd-config.values.yaml.gotmpl @@ -0,0 +1,2 @@ +environment: local + diff --git a/k8s/helmfile/env/production/argocd-config.values.yaml.gotmpl b/k8s/helmfile/env/production/argocd-config.values.yaml.gotmpl new file mode 100644 index 000000000..d3e915908 --- /dev/null +++ b/k8s/helmfile/env/production/argocd-config.values.yaml.gotmpl @@ -0,0 +1,2 @@ +environment: production + diff --git a/k8s/helmfile/env/staging/argocd-config.values.yaml.gotmpl b/k8s/helmfile/env/staging/argocd-config.values.yaml.gotmpl new file mode 100644 index 000000000..21e54ef69 --- /dev/null +++ b/k8s/helmfile/env/staging/argocd-config.values.yaml.gotmpl @@ -0,0 +1,2 @@ +environment: staging + diff --git a/k8s/helmfile/helmfile.yaml b/k8s/helmfile/helmfile.yaml index cfd49839b..8f2ddd5eb 100644 --- a/k8s/helmfile/helmfile.yaml +++ b/k8s/helmfile/helmfile.yaml @@ -104,6 +104,12 @@ releases: ################################ # ALL ENVIRONMENTS ################################ + - name: argocd-config + namespace: argocd + chart: ../../charts/argocd-config + installed: {{ eq .Environment.Name "local" | toYaml }} + <<: *default_release + - name: redirects namespace: default chart: ./../../charts/redirects @@ -159,6 +165,7 @@ releases: <<: *default_release - name: ui + installed: {{ ne .Environment.Name "local" | toYaml }} namespace: default chart: wbstack/ui version: 0.3.1