From 73a3966872a3d32e5c9353fb0bc586e0e00a8a05 Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Mon, 9 Dec 2024 17:01:55 +0100 Subject: [PATCH] Decouple automated K8s deployments This change allows deploying nativelink in various new configurations, including cache-only and multi-toolchain setups. The new setup is significantly more modular and lets us add and remove toolchains at runtime. Includes a reordering of the deployment logic to ensure that Alerts get deployed before the GitRepository that triggers them. For demo and CI this comes at the cost of an additional dummy GitRepository. This tradeoff seems well worth it as the previous ordering was entirely wrong and unreliable. Fixes the flakiness of the LRE workflow where it often gets stuck in the `Waiting for Tekton pipelines` loop. --- .github/workflows/lre.yaml | 72 +++++----- deploy/chromium-example/kustomization.yaml | 20 +-- deploy/dev/kustomization.yaml | 42 +++--- deploy/kubernetes-example/kustomization.yaml | 20 +-- kubernetes/base/kustomization.yaml | 12 -- kubernetes/components/alerts/alert.yaml | 17 +++ .../kustomization.yaml | 3 +- .../kustomization/flux-kustomization.yaml | 22 +++ .../kustomization.yaml | 4 +- .../components/operator/flux-config.yaml | 132 ------------------ .../components/worker/kustomization.yaml | 18 +++ .../worker}/worker.json5 | 0 .../{base => components/worker}/worker.yaml | 0 .../kustomization.yaml | 13 +- .../nativelink-config.json5 | 0 .../{base => nativelink}/nativelink.yaml | 0 .../overlays/chromium/kustomization.yaml | 22 +-- .../overlays/lre-manual/kustomization.yaml | 29 ++++ kubernetes/overlays/lre/kustomization.yaml | 28 +--- .../alert-kustomization.yaml | 24 ++++ .../alert-kustomization/kustomization.yaml | 6 + .../resources/flux/flux-kustomizations.yaml | 38 +++++ kubernetes/resources/flux/gitrepository.yaml | 11 ++ kubernetes/resources/flux/kustomization.yaml | 7 + .../gateway-routes/kustomization.yaml | 6 + .../gateway-routes/routes.yaml | 0 .../example-do-not-use-in-prod-key.pem | 0 .../example-do-not-use-in-prod-rootca.crt | 0 .../insecure-certs/kustomization.yaml | 4 +- .../nativelink-core/kustomization.yaml | 52 +++++++ .../resources/provider/kustomization.yaml | 6 + kubernetes/resources/provider/provider.yaml | 9 ++ kubernetes/workers/lre-cc/kustomization.yaml | 17 +++ .../lre => workers/lre-cc}/worker-lre-cc.yaml | 2 +- .../resources/lre-cc/kustomization.yaml | 60 ++++++++ .../siso-chromium/kustomization.yaml | 60 ++++++++ .../resources/worker-init/kustomization.yaml | 22 +++ .../workers/siso-chromium/kustomization.yaml | 11 ++ tools/pre-commit-hooks.nix | 2 +- .../docs/deployment-examples/chromium.mdx | 18 +++ .../docs/deployment-examples/kubernetes.mdx | 18 +++ 41 files changed, 536 insertions(+), 291 deletions(-) delete mode 100644 kubernetes/base/kustomization.yaml create mode 100644 kubernetes/components/alerts/alert.yaml rename kubernetes/components/{gateway-routes => alerts}/kustomization.yaml (82%) create mode 100644 kubernetes/components/kustomization/flux-kustomization.yaml rename kubernetes/components/{operator => kustomization}/kustomization.yaml (68%) delete mode 100644 kubernetes/components/operator/flux-config.yaml create mode 100644 kubernetes/components/worker/kustomization.yaml rename kubernetes/{configmaps => components/worker}/worker.json5 (100%) rename kubernetes/{base => components/worker}/worker.yaml (100%) rename kubernetes/{configmaps => nativelink}/kustomization.yaml (64%) rename kubernetes/{configmaps => nativelink}/nativelink-config.json5 (100%) rename kubernetes/{base => nativelink}/nativelink.yaml (100%) create mode 100644 kubernetes/overlays/lre-manual/kustomization.yaml create mode 100644 kubernetes/resources/alert-kustomization/alert-kustomization.yaml create mode 100644 kubernetes/resources/alert-kustomization/kustomization.yaml create mode 100644 kubernetes/resources/flux/flux-kustomizations.yaml create mode 100644 kubernetes/resources/flux/gitrepository.yaml create mode 100644 kubernetes/resources/flux/kustomization.yaml create mode 100644 kubernetes/resources/gateway-routes/kustomization.yaml rename kubernetes/{components => resources}/gateway-routes/routes.yaml (100%) rename kubernetes/{components => resources}/insecure-certs/example-do-not-use-in-prod-key.pem (100%) rename kubernetes/{components => resources}/insecure-certs/example-do-not-use-in-prod-rootca.crt (100%) rename kubernetes/{components => resources}/insecure-certs/kustomization.yaml (69%) create mode 100644 kubernetes/resources/nativelink-core/kustomization.yaml create mode 100644 kubernetes/resources/provider/kustomization.yaml create mode 100644 kubernetes/resources/provider/provider.yaml create mode 100644 kubernetes/workers/lre-cc/kustomization.yaml rename kubernetes/{overlays/lre => workers/lre-cc}/worker-lre-cc.yaml (98%) create mode 100644 kubernetes/workers/resources/lre-cc/kustomization.yaml create mode 100644 kubernetes/workers/resources/siso-chromium/kustomization.yaml create mode 100644 kubernetes/workers/resources/worker-init/kustomization.yaml create mode 100644 kubernetes/workers/siso-chromium/kustomization.yaml diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index 478b848f3..61ee94c46 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -96,16 +96,9 @@ jobs: nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - components: - - kubernetes/components/operator + resources: + - kubernetes/overlays/lre patches: - - patch: |- - - op: replace - path: /spec/path - value: ./kubernetes/overlays/lre - target: - kind: Kustomization - name: nativelink - patch: |- - op: replace path: /spec/url @@ -119,27 +112,6 @@ jobs: target: kind: GitRepository name: nativelink - - patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: ./src_root#image - target: - kind: Alert - name: nativelink-image-alert - - patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: ./src_root#nativelink-worker-init - target: - kind: Alert - name: nativelink-worker-init-alert - - patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: ./src_root#nativelink-worker-lre-cc - target: - kind: Alert - name: nativelink-worker-alert EOF kubectl apply -k . && rm kustomization.yaml' @@ -151,6 +123,36 @@ jobs: --timeout=15m \ nativelink-tekton-resources" + - name: Wait for alerts + run: > + nix develop --impure --command + bash -c "flux reconcile kustomization -n default \ + --timeout=15m \ + nativelink-alert-core && \ + flux reconcile kustomization -n default \ + --timeout=15m \ + nativelink-alert-worker-init && \ + flux reconcile kustomization -n default \ + --timeout=15m \ + nativelink-alert-lre-cc" + + - name: Trigger pipelines + run: | + nix develop --impure --command bash -c 'cat > dummy-repo.yaml << EOF + apiVersion: source.toolkit.fluxcd.io/v1 + kind: GitRepository + metadata: + name: dummy-repository + namespace: default + spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main + EOF + kubectl apply -f dummy-repo.yaml && + rm dummy-repo.yaml' + - name: Wait for Tekton pipelines run: > nix develop --impure --command bash << 'EOF' @@ -171,19 +173,19 @@ jobs: -l tekton.dev/pipeline=rebuild-nativelink EOF - - name: Wait for Configmaps + - name: Wait for NativeLink Kustomization run: > nix develop --impure --command bash -c "flux reconcile kustomization -n default \ --timeout=15m \ - nativelink-configmaps" + nativelink-core" - - name: Wait for NativeLink Kustomization + - name: Wait for Worker Kustomization run: > nix develop --impure --command bash -c "flux reconcile kustomization -n default \ --timeout=15m \ - nativelink" + nativelink-lre-cc" - name: Wait for NativeLink run: > @@ -193,7 +195,7 @@ jobs: - name: Wait for worker run: > nix develop --impure --command - bash -c "kubectl rollout status deploy/nativelink-worker" + bash -c "kubectl rollout status deploy/nativelink-worker-lre-cc" - name: Get gateway IPs id: gateway-ips diff --git a/deploy/chromium-example/kustomization.yaml b/deploy/chromium-example/kustomization.yaml index d8c45a9f9..d7da23572 100644 --- a/deploy/chromium-example/kustomization.yaml +++ b/deploy/chromium-example/kustomization.yaml @@ -1,21 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -components: -- ../../kubernetes/components/operator - -patches: -- patch: |- - - op: replace - path: /spec/path - value: ./kubernetes/overlays/chromium - target: - kind: Kustomization - name: nativelink -- patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium - target: - kind: Alert - name: nativelink-worker-alert +resources: +- ../../kubernetes/overlays/chromium diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml index 35a3f183a..976a5aaf5 100644 --- a/deploy/dev/kustomization.yaml +++ b/deploy/dev/kustomization.yaml @@ -1,33 +1,23 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -components: -- ../../kubernetes/components/operator +resources: +- ../../kubernetes/overlays/lre -# Change this value to deploy custom overlays. -patches: -- patch: |- - - op: replace - path: /spec/path - value: ./kubernetes/overlays/lre - target: - kind: Kustomization - name: nativelink - -# Modify this value to change the URL of the repository with deployment files. +# Modify this value to change the repository containing the deployment files. # # This is usually only necessary if you change deployment YAML files or # NativeLink config files. If you only intend to change the Rust sources you can # leave this as is and need to ensure that the Alerts below are patched to build # your local sources. +patches: - patch: |- - op: replace path: /spec/url value: https://github.com/TraceMachina/nativelink -# Optionally, change the tracked branch. -# - op: replace -# path: /spec/ref/branch -# value: somecustombranch + - op: replace + path: /spec/ref/branch + value: main target: kind: GitRepository name: nativelink @@ -43,22 +33,22 @@ patches: # outputs from a Pull request. - patch: |- - op: replace - path: /spec/eventMetadata/flakeOutput + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT value: ./src_root#image target: - kind: Alert - name: nativelink-image-alert + kind: Kustomization + name: nativelink-alert-core - patch: |- - op: replace - path: /spec/eventMetadata/flakeOutput + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT value: ./src_root#nativelink-worker-init target: - kind: Alert - name: nativelink-worker-init-alert + kind: Kustomization + name: nativelink-alert-worker-init - patch: |- - op: replace - path: /spec/eventMetadata/flakeOutput + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT value: ./src_root#nativelink-worker-lre-cc target: - kind: Alert - name: nativelink-worker-alert + kind: Kustomization + name: nativelink-alert-lre-cc diff --git a/deploy/kubernetes-example/kustomization.yaml b/deploy/kubernetes-example/kustomization.yaml index eb836d23e..343a1b90f 100644 --- a/deploy/kubernetes-example/kustomization.yaml +++ b/deploy/kubernetes-example/kustomization.yaml @@ -1,21 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -components: -- ../../kubernetes/components/operator - -patches: -- patch: |- - - op: replace - path: /spec/path - value: ./kubernetes/overlays/lre - target: - kind: Kustomization - name: nativelink -- patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: github:TraceMachina/nativelink#nativelink-worker-lre-cc - target: - kind: Alert - name: nativelink-worker-alert +resources: +- ../../kubernetes/overlays/lre diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml deleted file mode 100644 index 3a160e2ac..000000000 --- a/kubernetes/base/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - nativelink.yaml - - worker.yaml - - ../configmaps - -images: - - name: nativelink - - name: nativelink-worker-init - - name: nativelink-worker diff --git a/kubernetes/components/alerts/alert.yaml b/kubernetes/components/alerts/alert.yaml new file mode 100644 index 000000000..3450aa70b --- /dev/null +++ b/kubernetes/components/alerts/alert.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-alert + namespace: default +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: ${PLACEHOLDER_FLAKE_OUTPUT} + configMapName: ${PLACEHOLDER_CONFIG_MAP_NAME} diff --git a/kubernetes/components/gateway-routes/kustomization.yaml b/kubernetes/components/alerts/kustomization.yaml similarity index 82% rename from kubernetes/components/gateway-routes/kustomization.yaml rename to kubernetes/components/alerts/kustomization.yaml index 42b112b7e..1faa9eaf5 100644 --- a/kubernetes/components/gateway-routes/kustomization.yaml +++ b/kubernetes/components/alerts/kustomization.yaml @@ -1,5 +1,6 @@ --- apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component + resources: - - routes.yaml +- alert.yaml diff --git a/kubernetes/components/kustomization/flux-kustomization.yaml b/kubernetes/components/kustomization/flux-kustomization.yaml new file mode 100644 index 000000000..15d3f2034 --- /dev/null +++ b/kubernetes/components/kustomization/flux-kustomization.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + path: PLACEHOLDER_NATIVELINK_KUSTOMIZATION + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + postBuild: + substituteFrom: + - kind: ConfigMap + name: nativelink-image-tags diff --git a/kubernetes/components/operator/kustomization.yaml b/kubernetes/components/kustomization/kustomization.yaml similarity index 68% rename from kubernetes/components/operator/kustomization.yaml rename to kubernetes/components/kustomization/kustomization.yaml index 916d215e1..91474583a 100644 --- a/kubernetes/components/operator/kustomization.yaml +++ b/kubernetes/components/kustomization/kustomization.yaml @@ -1,4 +1,6 @@ +--- apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component + resources: -- flux-config.yaml + - flux-kustomization.yaml diff --git a/kubernetes/components/operator/flux-config.yaml b/kubernetes/components/operator/flux-config.yaml deleted file mode 100644 index 9b8b10fd8..000000000 --- a/kubernetes/components/operator/flux-config.yaml +++ /dev/null @@ -1,132 +0,0 @@ ---- -apiVersion: source.toolkit.fluxcd.io/v1 -kind: GitRepository -metadata: - name: nativelink - namespace: default -spec: - interval: 2m - url: https://github.com/TraceMachina/nativelink - ref: - branch: main ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Provider -metadata: - name: nativelink-webhook - namespace: flux-system -spec: - type: generic - address: http://el-nativelink-rebuild.default.svc.cluster.local:8080 ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Alert -metadata: - name: nativelink-image-alert - namespace: flux-system -spec: - eventSeverity: info - eventSources: - - kind: GitRepository - name: '*' - namespace: default - providerRef: - name: nativelink-webhook - eventMetadata: - flakeOutput: github:TraceMachina/nativelink#image - configMapName: nativelink-image-tags ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Alert -metadata: - name: nativelink-worker-init-alert - namespace: flux-system -spec: - eventSeverity: info - eventSources: - - kind: GitRepository - name: '*' - namespace: default - providerRef: - name: nativelink-webhook - eventMetadata: - flakeOutput: github:TraceMachina/nativelink#nativelink-worker-init - configMapName: nativelink-image-tags ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Alert -metadata: - name: nativelink-worker-alert - namespace: flux-system -spec: - eventSeverity: info - eventSources: - - kind: GitRepository - name: '*' - namespace: default - providerRef: - name: nativelink-webhook - eventMetadata: - flakeOutput: "PLACEHOLDER_NATIVELINK_WORKER" - configMapName: nativelink-image-tags ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: nativelink-configmaps - namespace: default -spec: - interval: 2m - path: "./kubernetes/configmaps" - prune: true - force: true - retryInterval: 20s - targetNamespace: default - wait: true - sourceRef: - kind: GitRepository - name: nativelink - namespace: default ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: nativelink - namespace: default -spec: - interval: 2m - path: "PLACEHOLDER_NATIVELINK_KUSTOMIZATION" - prune: true - force: true - retryInterval: 20s - targetNamespace: default - wait: true - sourceRef: - kind: GitRepository - name: nativelink - namespace: default - postBuild: - substituteFrom: - - kind: ConfigMap - name: nativelink-image-tags - dependsOn: - - name: nativelink-configmaps - - name: nativelink-tekton-resources ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: nativelink-tekton-resources - namespace: default -spec: - interval: 2m - path: "./kubernetes/resources/tekton" - prune: true - force: true - retryInterval: 20s - targetNamespace: default - wait: true - sourceRef: - kind: GitRepository - name: nativelink - namespace: default diff --git a/kubernetes/components/worker/kustomization.yaml b/kubernetes/components/worker/kustomization.yaml new file mode 100644 index 000000000..fccbe7f19 --- /dev/null +++ b/kubernetes/components/worker/kustomization.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- worker.yaml + +configMapGenerator: + - name: worker + files: + - worker.json5 + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + +images: + - name: nativelink-worker-init + - name: nativelink-worker diff --git a/kubernetes/configmaps/worker.json5 b/kubernetes/components/worker/worker.json5 similarity index 100% rename from kubernetes/configmaps/worker.json5 rename to kubernetes/components/worker/worker.json5 diff --git a/kubernetes/base/worker.yaml b/kubernetes/components/worker/worker.yaml similarity index 100% rename from kubernetes/base/worker.yaml rename to kubernetes/components/worker/worker.yaml diff --git a/kubernetes/configmaps/kustomization.yaml b/kubernetes/nativelink/kustomization.yaml similarity index 64% rename from kubernetes/configmaps/kustomization.yaml rename to kubernetes/nativelink/kustomization.yaml index 98d8c589d..db942f434 100644 --- a/kubernetes/configmaps/kustomization.yaml +++ b/kubernetes/nativelink/kustomization.yaml @@ -2,6 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +resources: + - nativelink.yaml + - ../resources/insecure-certs + configMapGenerator: - name: nativelink-config files: @@ -9,9 +13,6 @@ configMapGenerator: options: annotations: kustomize.toolkit.fluxcd.io/substitute: disabled - - name: worker - files: - - worker.json5 - options: - annotations: - kustomize.toolkit.fluxcd.io/substitute: disabled + +images: + - name: nativelink diff --git a/kubernetes/configmaps/nativelink-config.json5 b/kubernetes/nativelink/nativelink-config.json5 similarity index 100% rename from kubernetes/configmaps/nativelink-config.json5 rename to kubernetes/nativelink/nativelink-config.json5 diff --git a/kubernetes/base/nativelink.yaml b/kubernetes/nativelink/nativelink.yaml similarity index 100% rename from kubernetes/base/nativelink.yaml rename to kubernetes/nativelink/nativelink.yaml diff --git a/kubernetes/overlays/chromium/kustomization.yaml b/kubernetes/overlays/chromium/kustomization.yaml index 10837536b..7c95724f4 100644 --- a/kubernetes/overlays/chromium/kustomization.yaml +++ b/kubernetes/overlays/chromium/kustomization.yaml @@ -1,20 +1,10 @@ --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- ../../base - -components: -- ../../components/gateway-routes -- ../../components/insecure-certs -images: - - name: nativelink - newName: localhost:5001/nativelink - newTag: ${NATIVELINK_TAG} - - name: nativelink-worker-init - newName: localhost:5001/nativelink-worker-init - newTag: ${NATIVELINK_WORKER_INIT_TAG} - - name: nativelink-worker - newName: localhost:5001/nativelink-worker-siso-chromium - newTag: ${NATIVELINK_WORKER_SISO_CHROMIUM_TAG} +resources: +- ../../resources/flux +- ../../resources/nativelink-core +- ../../resources/gateway-routes +- ../../workers/resources/worker-init +- ../../workers/resources/siso-chromium diff --git a/kubernetes/overlays/lre-manual/kustomization.yaml b/kubernetes/overlays/lre-manual/kustomization.yaml new file mode 100644 index 000000000..e48b1f7b7 --- /dev/null +++ b/kubernetes/overlays/lre-manual/kustomization.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../nativelink + - ../../workers/lre-cc + - ../../resources/gateway-routes + +# Note: This deployment is mainly intended for documentation purposes. If you +# intend to use it you'll need to provide image tags here. +# +# NativeLink doesn't support a "latest" tag or similar for reproducibility +# reasons. For the latest tags see: +# https://github.com/orgs/TraceMachina/packages?repo_name=nativelink +# +# See the lre overlay for a setup that autodetects these tags and automatically +# stays in sync with main. See the `../deploy/dev` directory for development +# setups. +images: + - name: nativelink + newName: ghcr.io/TraceMachina/nativelink + newTag: TODO + - name: nativelink-worker-init + newName: ghcr.io/TraceMachina/nativelink-worker-init + newTag: TODO + - name: nativelink-worker-lre-cc + newName: ghcr.io/TraceMachina/nativelink-worker-lre-cc + newTag: TODO diff --git a/kubernetes/overlays/lre/kustomization.yaml b/kubernetes/overlays/lre/kustomization.yaml index 707804b4e..833d5bcfa 100644 --- a/kubernetes/overlays/lre/kustomization.yaml +++ b/kubernetes/overlays/lre/kustomization.yaml @@ -1,26 +1,10 @@ --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- ../../base - -components: -- ../../components/gateway-routes -- ../../components/insecure-certs -patches: -- path: worker-lre-cc.yaml - target: - kind: Deployment - name: nativelink-worker - -images: - - name: nativelink - newName: localhost:5001/nativelink - newTag: ${NATIVELINK_TAG} - - name: nativelink-worker-init - newName: localhost:5001/nativelink-worker-init - newTag: ${NATIVELINK_WORKER_INIT_TAG} - - name: nativelink-worker - newName: localhost:5001/nativelink-worker-lre-cc - newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} +resources: +- ../../resources/flux +- ../../resources/nativelink-core +- ../../resources/gateway-routes +- ../../workers/resources/worker-init +- ../../workers/resources/lre-cc diff --git a/kubernetes/resources/alert-kustomization/alert-kustomization.yaml b/kubernetes/resources/alert-kustomization/alert-kustomization.yaml new file mode 100644 index 000000000..470912758 --- /dev/null +++ b/kubernetes/resources/alert-kustomization/alert-kustomization.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink-alert + namespace: default +spec: + interval: 2m + path: ./kubernetes/components/alerts + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + postBuild: + substitute: + PLACEHOLDER_FLAKE_OUTPUT: SET_BY_FLUX + PLACEHOLDER_CONFIG_MAP_NAME: SET_BY_FLUX + dependsOn: + - name: provider diff --git a/kubernetes/resources/alert-kustomization/kustomization.yaml b/kubernetes/resources/alert-kustomization/kustomization.yaml new file mode 100644 index 000000000..e6922e332 --- /dev/null +++ b/kubernetes/resources/alert-kustomization/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - alert-kustomization.yaml diff --git a/kubernetes/resources/flux/flux-kustomizations.yaml b/kubernetes/resources/flux/flux-kustomizations.yaml new file mode 100644 index 000000000..9f23a41b5 --- /dev/null +++ b/kubernetes/resources/flux/flux-kustomizations.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: provider + namespace: default +spec: + interval: 2m + path: "./kubernetes/resources/provider" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + dependsOn: + - name: nativelink-tekton-resources +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink-tekton-resources + namespace: default +spec: + interval: 2m + path: "./kubernetes/resources/tekton" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default diff --git a/kubernetes/resources/flux/gitrepository.yaml b/kubernetes/resources/flux/gitrepository.yaml new file mode 100644 index 000000000..cdf6a2a5a --- /dev/null +++ b/kubernetes/resources/flux/gitrepository.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main diff --git a/kubernetes/resources/flux/kustomization.yaml b/kubernetes/resources/flux/kustomization.yaml new file mode 100644 index 000000000..474172b82 --- /dev/null +++ b/kubernetes/resources/flux/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - gitrepository.yaml + - flux-kustomizations.yaml diff --git a/kubernetes/resources/gateway-routes/kustomization.yaml b/kubernetes/resources/gateway-routes/kustomization.yaml new file mode 100644 index 000000000..4e8451152 --- /dev/null +++ b/kubernetes/resources/gateway-routes/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - routes.yaml diff --git a/kubernetes/components/gateway-routes/routes.yaml b/kubernetes/resources/gateway-routes/routes.yaml similarity index 100% rename from kubernetes/components/gateway-routes/routes.yaml rename to kubernetes/resources/gateway-routes/routes.yaml diff --git a/kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem b/kubernetes/resources/insecure-certs/example-do-not-use-in-prod-key.pem similarity index 100% rename from kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem rename to kubernetes/resources/insecure-certs/example-do-not-use-in-prod-key.pem diff --git a/kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt b/kubernetes/resources/insecure-certs/example-do-not-use-in-prod-rootca.crt similarity index 100% rename from kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt rename to kubernetes/resources/insecure-certs/example-do-not-use-in-prod-rootca.crt diff --git a/kubernetes/components/insecure-certs/kustomization.yaml b/kubernetes/resources/insecure-certs/kustomization.yaml similarity index 69% rename from kubernetes/components/insecure-certs/kustomization.yaml rename to kubernetes/resources/insecure-certs/kustomization.yaml index bc92cb026..203e442c1 100644 --- a/kubernetes/components/insecure-certs/kustomization.yaml +++ b/kubernetes/resources/insecure-certs/kustomization.yaml @@ -1,6 +1,6 @@ --- -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization secretGenerator: - name: tls-secret diff --git a/kubernetes/resources/nativelink-core/kustomization.yaml b/kubernetes/resources/nativelink-core/kustomization.yaml new file mode 100644 index 000000000..a4aa1b2ea --- /dev/null +++ b/kubernetes/resources/nativelink-core/kustomization.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -core + +resources: + - ../../resources/alert-kustomization + +components: + - ../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#image + - op: add + path: /spec/nameSuffix + value: -core + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/nativelink + - op: add + path: /spec/images + value: + - name: nativelink + newName: localhost:5001/nativelink + newTag: ${NATIVELINK_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/resources/provider/kustomization.yaml b/kubernetes/resources/provider/kustomization.yaml new file mode 100644 index 000000000..353ca7f5d --- /dev/null +++ b/kubernetes/resources/provider/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - provider.yaml diff --git a/kubernetes/resources/provider/provider.yaml b/kubernetes/resources/provider/provider.yaml new file mode 100644 index 000000000..b99e8f4af --- /dev/null +++ b/kubernetes/resources/provider/provider.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: nativelink-webhook + namespace: default +spec: + type: generic + address: http://el-nativelink-rebuild.default.svc.cluster.local:8080 diff --git a/kubernetes/workers/lre-cc/kustomization.yaml b/kubernetes/workers/lre-cc/kustomization.yaml new file mode 100644 index 000000000..a73b9c052 --- /dev/null +++ b/kubernetes/workers/lre-cc/kustomization.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -lre-cc + +components: +- ../../components/worker + +patches: +- path: worker-lre-cc.yaml + target: + kind: Deployment + name: nativelink-worker + +images: +- name: nativelink-worker + newName: nativelink-worker-lre-cc diff --git a/kubernetes/overlays/lre/worker-lre-cc.yaml b/kubernetes/workers/lre-cc/worker-lre-cc.yaml similarity index 98% rename from kubernetes/overlays/lre/worker-lre-cc.yaml rename to kubernetes/workers/lre-cc/worker-lre-cc.yaml index 6363d91ec..654b90b66 100644 --- a/kubernetes/overlays/lre/worker-lre-cc.yaml +++ b/kubernetes/workers/lre-cc/worker-lre-cc.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-worker + name: nativelink-worker-lre-cc spec: replicas: 1 template: diff --git a/kubernetes/workers/resources/lre-cc/kustomization.yaml b/kubernetes/workers/resources/lre-cc/kustomization.yaml new file mode 100644 index 000000000..f96708225 --- /dev/null +++ b/kubernetes/workers/resources/lre-cc/kustomization.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -lre-cc + +resources: + - ../../../resources/alert-kustomization + +components: + - ../../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-lre-cc + - op: add + path: /spec/nameSuffix + value: -lre-cc + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/workers/lre-cc + - op: add + path: /spec/postBuild/substituteFrom/1 + value: + kind: ConfigMap + name: nativelink-worker-init + - op: add + path: /spec/images + value: + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker-lre-cc + newName: localhost:5001/nativelink-worker-lre-cc + newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/workers/resources/siso-chromium/kustomization.yaml b/kubernetes/workers/resources/siso-chromium/kustomization.yaml new file mode 100644 index 000000000..53b83f768 --- /dev/null +++ b/kubernetes/workers/resources/siso-chromium/kustomization.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -siso-chromium + +resources: + - ../../../resources/alert-kustomization + +components: + - ../../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium + - op: add + path: /spec/nameSuffix + value: -siso-chromium + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/workers/siso-chromium + - op: add + path: /spec/postBuild/substituteFrom/1 + value: + kind: ConfigMap + name: nativelink-worker-init + - op: add + path: /spec/images + value: + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker-siso-chromium + newName: localhost:5001/nativelink-worker-siso-chromium + newTag: ${NATIVELINK_WORKER_SISO_CHROMIUM_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/workers/resources/worker-init/kustomization.yaml b/kubernetes/workers/resources/worker-init/kustomization.yaml new file mode 100644 index 000000000..ee4559366 --- /dev/null +++ b/kubernetes/workers/resources/worker-init/kustomization.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -worker-init + +resources: + - ../../../resources/alert-kustomization + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-init + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_CONFIG_MAP_NAME + value: nativelink-worker-init + - op: add + path: /spec/nameSuffix + value: -worker-init + target: + kind: Kustomization + name: nativelink-alert diff --git a/kubernetes/workers/siso-chromium/kustomization.yaml b/kubernetes/workers/siso-chromium/kustomization.yaml new file mode 100644 index 000000000..babad5ade --- /dev/null +++ b/kubernetes/workers/siso-chromium/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -siso-chromium + +components: +- ../../components/worker + +images: +- name: nativelink-worker + newName: nativelink-worker-siso-chromium diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index 77620b080..f47277ac4 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -48,7 +48,7 @@ in { ++ [ # Integration testfiles not intended for production. "deployment-examples/docker-compose/example-do-not-use-in-prod-key.pem" - "kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem" + "kubernetes/resources/insecure-certs/example-do-not-use-in-prod-key.pem" ]; enable = true; name = "detect-private-key"; diff --git a/web/platform/src/content/docs/docs/deployment-examples/chromium.mdx b/web/platform/src/content/docs/docs/deployment-examples/chromium.mdx index 021512c19..41863bfe1 100644 --- a/web/platform/src/content/docs/docs/deployment-examples/chromium.mdx +++ b/web/platform/src/content/docs/docs/deployment-examples/chromium.mdx @@ -36,6 +36,24 @@ Now, start the development cluster: native up ``` +Once the infra is ready, trigger the pipelines that build the images: + +```bash +cat > dummy-repo.yaml << EOF +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: dummy-repository + namespace: default +spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main +EOF +kubectl apply -f dummy-repo.yaml +``` + :::tip The `native up` command uses Pulumi under the hood. You can view and delete the stack with `pulumi stack` and `pulumi destroy`. If you're queried for a diff --git a/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx b/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx index 34caf3ab9..a2e927b58 100644 --- a/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx +++ b/web/platform/src/content/docs/docs/deployment-examples/kubernetes.mdx @@ -50,6 +50,24 @@ kubectl apply -k \ https://github.com/TraceMachina/nativelink//deploy/kubernetes-example ``` +Once the infra is ready, trigger the pipelines that build the images: + +```bash +cat > dummy-repo.yaml << EOF +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: dummy-repository + namespace: default +spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main +EOF +kubectl apply -f dummy-repo.yaml +``` + :::danger This example is built for demo purposes only. It's not a secure production-grade setup and will only work in the local development cluster created with