refacto: add cluster-api and workload (#18)

* clean

* fix test

* module like

* module like

.github/workflows/test.yaml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         # kubernetes releases
-        k8s: [v1.27.11, v1.28.7, v1.29.2]
+        k8s: [v1.27.13, v1.28.9, v1.29.4]
 
     runs-on: ubuntu-latest
 
@@ -29,8 +29,11 @@ jobs:
 
       - name: lint
         run: |
-          echo "lint files"
-          time make lint
+          echo "Install local root CA"
+          time make local-root-ca
+          echo "Lint files"
+          time make lint HELMFILE_FILE=helmfile.d/cluster-configure-core
+          time make lint HELMFILE_FILE=helmfile.d/cluster-workload
 
       - name: "Create local kubernetes ${{ matrix.k8s }} cluster"
         run: |
@@ -39,15 +42,35 @@ jobs:
 
       - name: Test on local environment
         run: |
-          echo "Install local root CA"
-          time make local-root-ca
           echo "Sync releases"
-          time make apply
+          time make apply HELMFILE_FILE=helmfile.d/cluster-configure-core
+          echo "Wait releases"
+          timeout=5
+          interval=1
+          test_result=1
+          set +e
+          until [ "$timeout" -le 0 -o "$test_result" -eq "0" ] ; do
+             time make apply HELMFILE_FILE=helmfile.d/cluster-workload
+             test_result=$?
+             if [ "$test_result" -gt 0 ] ;then
+               echo "workload is not accessible yet. Retrying in $interval/$timeout seconds: $test_result";
+               (( timeout-- ))
+               sleep $interval
+             fi
+          done
+          set -e
+          if [ "$test_result" -gt 0 ] ;then
+            test_status=ERROR
+            echo "$test_status: Timeout reached. workload is not accessible. result: $test_result"
+            return $test_result
+          fi
           helm list -A
+          kubectl get pod -A
           echo "Test local url"
           time make ci-local-tests
           echo "Destroy resources"
-          time make destroy
+          time make destroy HELMFILE_FILE=helmfile.d/cluster-workload
+          time make destroy HELMFILE_FILE=helmfile.d/cluster-configure-core
 
       - name: Teardown
         if: always()

.sops.yaml

@@ -0,0 +1,5 @@
+creation_rules:
+  - age: >-
+      age1wg6z592fc3l6u8gceyqptqyy8l4eg76yq5987ffj8wn6hur0v39qdvvkd0,
+      age1e4m0g70zk6gry3mh7xwlsnyxxdm3rxpwghkwgtw4ece6mqv53vysdmkh4v,
+      age13xyyu756vdwjfq9qe66uae5rr9waytvgj8xhne46l0kuj0ad23nsnr06gp

Makefile

@@ -1,21 +1,22 @@
 HELMFILE_ENVIRONMENT := local
-K8S_VERSION=v1.29.2
+HELMFILE_FILE := helmfile.d
+K8S_VERSION=v1.29.4
 export
 
 lint: repos
-	@helmfile -e $(HELMFILE_ENVIRONMENT) lint
+	@helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) lint
 repos:
-	@helmfile -e $(HELMFILE_ENVIRONMENT) repos
+	@helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) repos
 template:
-	@helmfile -e $(HELMFILE_ENVIRONMENT) template --include-crds  --include-needs  --include-transitive-needs -q
+	@helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) template --include-crds  --include-needs  --include-transitive-needs
 diff:
-	@helmfile -e $(HELMFILE_ENVIRONMENT) diff
+	@helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) diff
 sync:
-	helmfile -e $(HELMFILE_ENVIRONMENT) sync
+	helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) sync
 apply:
-	helmfile -e $(HELMFILE_ENVIRONMENT) apply --skip-diff-on-install
+	helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) apply --skip-diff-on-install
 destroy:
-	helmfile -e $(HELMFILE_ENVIRONMENT) destroy
+	helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE) destroy
 
 local-root-ca:
 	@echo "# import local root CA"
@@ -46,10 +47,13 @@ ci-delete-docker-registry:
 	docker rm -f  kind-registry || true
 
 # Boostrap minimal core apps (argocd) and app-of-apps
-HELMFILE_FILE_CORE=helmfile.d/01-core-apps.yaml
-HELMFILE_FILE_ARGOCD=helmfile.d/02-argocd.yaml
+HELMFILE_FILE_CORE=helmfile.d/cluster-configure-core
+HELMFILE_FILE_ARGOCD=helmfile.d/cluster-workload/10-argocd.yaml
+export HELMFILE_FILE_CORE
 
-boostrap-core:
+bootstrap-core: HELMFILE_FILE=$(HELMFILE_FILE_CORE)
+bootstrap-core:
 	helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE_CORE) apply --skip-diff-on-install
-boostrap-argocd: boostrap-core
+bootstrap-argocd: HELMFILE_FILE=$(HELMFILE_FILE_ARGOCD)
+bootstrap-argocd: bootstrap-core
 	helmfile -e $(HELMFILE_ENVIRONMENT) -f $(HELMFILE_FILE_ARGOCD) apply --skip-diff-on-install

README.md

@@ -24,82 +24,104 @@ A release is a deployment of a Helm chart in an environment, where an environmen
 the combination of a set of parameters ("values") and a Kubernetes target.
 
 - Helmfile is a management system for Helm releases.
-It allows the mapping of 'charts' to 'values' to be declared in files.
+It allows the mapping of 'charts' to 'values' and 'secrets' to be declared in files.
 values can be declared at various levels (environments, services, role...), as template or raw values
 
 - Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
 It use Git repositories as the source of truth for defining the desired application state. It also allows to deploy Helm charts or deploy with helmfile.
 
+Note:
+argocd run the helmfile plugin. In this context, helmfile generate template to output.
+Rendered ressources are send to kubernetes cluster in stream
+helmfile is not executed directly in argocd
+
 ## Repository Structure
+It is organized as followed
+- apps: argocd app per cluster and steps for cluster
+- helmfile.d: helmfile releases to create and configure cluster. generic helmfile and helm charts with default configuration
+- environments: per environment and cluster configuration and secrets (can be hosted on private repo)
+
+A local environment is provided to develop on local machine
+
 ### helmfile.d
 
-A top-level helmfile directory, contains dependencies release files and directory.
+A top-level helmfile directory, contains directories and dependencies release files
 All the yaml files under the specified directory are processed in the alphabetical order.
 Each files defines an ordered list of  releases to deploy.
 
-- 01-core-apps.yaml: for core applications (example ingress,cert-manager,...)
-- 02-argocd.yaml: argocd and app-of-apps
-- 03-loki.yaml: logs aggregator
-- 02-prometheus-stack.yaml: observability (grafana,prometheus)
-- 03-promtail.yaml: logs shipping
-- 02-sample-apps.yaml: other applications
+- cluster-bootstrap: cluster-api
+  - 01-create-cluster.yaml : create kubernetes cluster with cluster-api
+
+- cluster-configure-core: core applications (ingress,cert-manager,...)
+  - 00-namespaces.yaml
+  - 01-cert-manager.yaml
+  - 02-ingress-nginx.yaml
 
+- cluster-workload: addons workload applications (argocd,prometheus,loki,...)
+  - 00-namespaces-workload.yaml
+  - 10-argocd.yaml
+  - 30-loki.yaml
+  - 31-prometheus-stack.yaml
+  - 32-promtail.yaml
+  - 80-whoami.yaml
 
 #### bases
 This directory contains 2 files:
 - `helmDefaults.yaml`, which directs `helmfile` on global configuration to pass to `helm`
 - and `environments.yaml` wich defines defines various environments (define in environments directory)
 
-#### environments
-This directory contains values for each environments, and kubernetes target
+#### base/environments
+This directory contains default values for all environments, and local target
 
 Files are loaded in following order and the last override the first.
-- `common.yaml`: is the common file loaded in all environement
-- `{{ environment }}.yaml`
 
 Environments:
 - `default.yaml`: is the default use by helmfile without any argument
-- `local.yaml`: is a sample local environment, (based on KinD kubernetes target for example)
-- `dev.yaml`: is an other sample dev environment
+- `local.yaml`: is a sample local environment, (based on KinD kubernetes target for development)
 
-```
-# example:  to deploy one environment, for example 'local' use:
-helmfile -e local  sync
-```
 
 #### releases
 Each subdirectory of `releases` contains 1 (or sometimes more!) release.
 
 Each helmfile in releases, define helm repository and releases dependencies (helm charts), and values and secrets (for helm charts)
 
-- `releases/01-core/`: contains minimal workload needed on kubernetes target (ingress-nginx, observability stack)
+- `releases/cert-manager/`
+- `releases/ingress-nginx/`
 - `releases/argocd/`
 - `releases/loki/`
 - `releases/promtail/`
 - `releases/prometheus-stack/`
-- `releases/sample-whoami/`: contains sample app workload, whoami
+- `releases/whoami/`: contains sample app workload, whoami
 
-#### Values
+#### Values and Secrets
 
 Values for releases can be defined in following order:
-- lookup a value file named `Release_Name-values.gotmpl`
-- lookup a value file named `env/Environment_Name/Release_Name-values.gotmpl`
+- lookup a value file named `releases/Relase_name/Release_Name-values.gotmpl`
+- lookup a value file named `../environnements/Environment_Name/Release_name/values.gotmpl`
 
 ```
   ...
   values:
     - "{{`{{ .Release.Name }}`}}-values.yaml.gotmpl"
-    - "env/{{ .Environment.Name }}/{{`{{ .Release.Name }}`}}-values.yaml.gotmpl"
+    - "env/{{ .Environment.Name }}/{{`{{ .Release.Name }}`}}/values.yaml.gotmpl"
 
 Secrets can be defined in the same way
 ```
 
+### environments
+This directory contains least values loaded for all environments
+
+environments:
+  - ENV.yaml
+  - ENV/{releases}/values.yaml
+  - ENV/{releases}/secrets.enc.yaml
+
 ### ArgoCD app of apps
 
 ArgoCD can deploy helmfiles with the concept of app of apps
 
 For this example:
-- We define the git repository which contain the root apps (the apps of apps) in `helmfile.d/releases/argocd/env/local/extra-argocd-values.yaml.gotmpl`
+- We define the git repository which contain the root apps (the apps of apps) in `environments/local/extra-argocd/values.yaml.gotmpl`
   - in this demo: argocd watch this repo and is trigger on branch `test-argocd-helmfile`
 - Then, we define all Argocd Application in `argocd-apps/app-helmfile*.yaml`
 - Argocd will trigger all deployment on Every commit, on the branch
@@ -129,21 +151,6 @@ If needed, a Makefile is available as a wrapper to helmfile and "local" environm
 
 Some usefull targets of the Makefile:
 
-Some targets of makefile:
-
-- Lint files
-```
-make lint
-# or
-helmfile -e local lint
-```
-- Display templated files
-```
-make template
-# or
-helmfile -e local template
-```
-
 - Install local kind cluster
 ```
 # kind cluster
@@ -157,20 +164,24 @@ make ci-bootstrap-local-cluster-with-registry
 make local-root-ca
 ```
 
-- First deployment (needed to load some CRDS)
+- Lint files
+```
+make lint HELMFILE_FILE=helmfile.d/cluster-configure-core
+```
+- Display templated files
+```
+make template HELMFILE_FILE=helmfile.d/cluster-configure-core
+```
 
+- First deployment (needed to load some CRDS)
 ```
 # first deployment, needed to load CRDS
-make sync
-# or
-helmfile -e local sync
+make sync HELMFILE_FILE=helmfile.d/cluster-configure-core
 ```
 
 - Deploy all helmfiles
 ```
-make apply
-# or
-helmfile -e local apply
+make apply HELMFILE_FILE=helmfile.d/cluster-configure-core
 ```
 - Deployment of kind cluster with only core apps (ingres-nginx,cert-manager)
 ```
@@ -185,13 +196,22 @@ make boostrap-argocd
 - Diff mode only
 ```
 make diff
-# or
-helmfile -e local diff
 ```
 
 Destroy all resources
 ```
 make destroy
-# or
-helmfile -e local destroy
+```
+
+### test helmfile for another environment: lint and template
+
+```
+# lint file
+( export CLUSTER_NAME=c1-demo ; make  HELMFILE_ENVIRONMENT=${CLUSTER_NAME}  HELMFILE_FILE=helmfile.d/cluster-configure-core lint )
+
+# generate template
+( export CLUSTER_NAME=c1-demo ; make  HELMFILE_ENVIRONMENT=${CLUSTER_NAME}  HELMFILE_FILE=helmfile.d/cluster-configure-core template )
+
+# test template in dry run on antoher real cluster
+( export CLUSTER_NAME=c1-demo ; export KUBECONFIG=$HOME/.kube/${CLUSTER_NAME}.kubeconfig ; make  HELMFILE_ENVIRONMENT=${CLUSTER_NAME}  HELMFILE_FILE=helmfile.d/workload-cluster/  KUBECONFIG=$KUBECONFIG template | kubectl apply -f e --dry-run=client )
 ```

apps/cluster-c1-demo-core.yaml

@@ -0,0 +1,42 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  # here is the argocd app name for the cluster
+  name: cluster-c1-demo-core
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  destination:
+    # here is the name of the new workload cluster
+    name: c1-demo
+  source:
+    # replace with private repo , branch and path
+    repoURL: https://github.com/pli01/k8s-helmfiles.git
+    targetRevision: argocd
+    path: .
+    plugin:
+      name: helmfile-v1.0
+      env:
+        # here is the cluster name to configure
+        - name: ENV_NAME
+          value: "c1-demo"
+        - name: HELMFILE_FILE
+          value: "helmfile.d/cluster-configure-core"
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+#    automated:
+#      prune: true
+#      selfHeal: true
+#      allowEmpty: true
+#    retry:
+#      limit: 5
+#      backoff:
+#        duration: 5s
+#        maxDuration: 3m0s
+#        factor: 2