Decouple automated K8s deployments

This change allows deploying nativelink in various new configurations,
including cache-only and multi-toolchain setups.

The new setup is significantly more modular and lets us add and remove
toolchains at runtime.

Includes a reordering of the deployment logic to ensure that Alerts get
deployed before the GitRepository that triggers them. For demo and CI
this comes at the cost of an additional dummy GitRepository. This
tradeoff seems well worth it as the previous ordering was entirely wrong
and unreliable.

Fixes the flakiness of the LRE workflow where it often gets stuck in the
`Waiting for Tekton pipelines` loop.

.github/workflows/lre.yaml

@@ -96,16 +96,9 @@ jobs:
           nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF
           apiVersion: kustomize.config.k8s.io/v1beta1
           kind: Kustomization
-          components:
-          - kubernetes/components/operator
+          resources:
+          - kubernetes/overlays/lre
           patches:
-          - patch: |-
-              - op: replace
-                path: /spec/path
-                value: ./kubernetes/overlays/lre
-            target:
-              kind: Kustomization
-              name: nativelink
           - patch: |-
               - op: replace
                 path: /spec/url
@@ -119,27 +112,6 @@ jobs:
             target:
               kind: GitRepository
               name: nativelink
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#image
-            target:
-              kind: Alert
-              name: nativelink-image-alert
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#nativelink-worker-init
-            target:
-              kind: Alert
-              name: nativelink-worker-init-alert
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#nativelink-worker-lre-cc
-            target:
-              kind: Alert
-              name: nativelink-worker-alert
           EOF
           kubectl apply -k . &&
           rm kustomization.yaml'
@@ -151,6 +123,36 @@ jobs:
             --timeout=15m \
             nativelink-tekton-resources"
 
+      - name: Wait for alerts
+        run: >
+          nix develop --impure --command
+          bash -c "flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-core && \
+          flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-worker-init && \
+          flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-lre-cc"
+
+      - name: Trigger pipelines
+        run: |
+          nix develop --impure --command bash -c 'cat > dummy-repo.yaml << EOF
+          apiVersion: source.toolkit.fluxcd.io/v1
+          kind: GitRepository
+          metadata:
+            name: dummy-repository
+            namespace: default
+          spec:
+            interval: 2m
+            url: https://github.com/TraceMachina/nativelink
+            ref:
+              branch: main
+          EOF
+          kubectl apply -f dummy-repo.yaml &&
+          rm dummy-repo.yaml'
+
       - name: Wait for Tekton pipelines
         run: >
           nix develop --impure --command bash << 'EOF'
@@ -171,19 +173,19 @@ jobs:
               -l tekton.dev/pipeline=rebuild-nativelink
           EOF
 
-      - name: Wait for Configmaps
+      - name: Wait for NativeLink Kustomization
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink-configmaps"
+            nativelink-core"
 
-      - name: Wait for NativeLink Kustomization
+      - name: Wait for Worker Kustomization
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink"
+            nativelink-lre-cc"
 
       - name: Wait for NativeLink
         run: >
@@ -193,7 +195,7 @@ jobs:
       - name: Wait for worker
         run: >
           nix develop --impure --command
-          bash -c "kubectl rollout status deploy/nativelink-worker"
+          bash -c "kubectl rollout status deploy/nativelink-worker-lre-cc"
 
       - name: Get gateway IPs
         id: gateway-ips

deploy/chromium-example/kustomization.yaml

@@ -1,21 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
-
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/chromium
-  target:
-    kind: Kustomization
-    name: nativelink
-- patch: |-
-    - op: replace
-      path: /spec/eventMetadata/flakeOutput
-      value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium
-  target:
-    kind: Alert
-    name: nativelink-worker-alert
+resources:
+- ../../kubernetes/overlays/chromium

deploy/dev/kustomization.yaml

@@ -1,33 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
+resources:
+- ../../kubernetes/overlays/lre
 
-# Change this value to deploy custom overlays.
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/lre
-  target:
-    kind: Kustomization
-    name: nativelink
-
-# Modify this value to change the URL of the repository with deployment files.
+# Modify this value to change the repository containing the deployment files.
 #
 # This is usually only necessary if you change deployment YAML files or
 # NativeLink config files. If you only intend to change the Rust sources you can
 # leave this as is and need to ensure that the Alerts below are patched to build
 # your local sources.
+patches:
 - patch: |-
     - op: replace
       path: /spec/url
       value: https://github.com/TraceMachina/nativelink
-# Optionally, change the tracked branch.
-#   - op: replace
-#     path: /spec/ref/branch
-#     value: somecustombranch
+    - op: replace
+      path: /spec/ref/branch
+      value: main
   target:
     kind: GitRepository
     name: nativelink
@@ -43,22 +33,22 @@ patches:
 # outputs from a Pull request.
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#image
   target:
-    kind: Alert
-    name: nativelink-image-alert
+    kind: Kustomization
+    name: nativelink-alert-core
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#nativelink-worker-init
   target:
-    kind: Alert
-    name: nativelink-worker-init-alert
+    kind: Kustomization
+    name: nativelink-alert-worker-init
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#nativelink-worker-lre-cc
   target:
-    kind: Alert
-    name: nativelink-worker-alert
+    kind: Kustomization
+    name: nativelink-alert-lre-cc

deploy/kubernetes-example/kustomization.yaml

@@ -1,21 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
-
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/lre
-  target:
-    kind: Kustomization
-    name: nativelink
-- patch: |-
-    - op: replace
-      path: /spec/eventMetadata/flakeOutput
-      value: github:TraceMachina/nativelink#nativelink-worker-lre-cc
-  target:
-    kind: Alert
-    name: nativelink-worker-alert
+resources:
+- ../../kubernetes/overlays/lre

kubernetes/components/alerts/alert.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Alert
+metadata:
+  name: nativelink-alert
+  namespace: default
+spec:
+  eventSeverity: info
+  eventSources:
+    - kind: GitRepository
+      name: '*'
+      namespace: default
+  providerRef:
+    name: nativelink-webhook
+  eventMetadata:
+    flakeOutput: ${PLACEHOLDER_FLAKE_OUTPUT}
+    configMapName: ${PLACEHOLDER_CONFIG_MAP_NAME}

kubernetes/components/gateway-routes/kustomization.yaml → kubernetes/components/alerts/kustomization.yaml

@@ -1,5 +1,6 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
+
 resources:
-  - routes.yaml
+- alert.yaml

kubernetes/components/kustomization/flux-kustomization.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nativelink
+  namespace: default
+spec:
+  interval: 2m
+  path: PLACEHOLDER_NATIVELINK_KUSTOMIZATION
+  prune: true
+  force: true
+  retryInterval: 20s
+  targetNamespace: default
+  wait: true
+  sourceRef:
+    kind: GitRepository
+    name: nativelink
+    namespace: default
+  postBuild:
+    substituteFrom:
+      - kind: ConfigMap
+        name: nativelink-image-tags

kubernetes/components/operator/kustomization.yaml → kubernetes/components/kustomization/kustomization.yaml

@@ -1,4 +1,6 @@
+---
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component
+
 resources:
-- flux-config.yaml
+  - flux-kustomization.yaml