diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 14083e8cc4..351d87878a 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -5,6 +5,7 @@ Cloudflare ELB GPUs Goma +Kustomization LLD LLVM Machina diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index bdcab0adf4..cf5f76d11a 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -78,19 +78,38 @@ jobs: uses: >- # v4 DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41 - - name: Start Kubernetes cluster (Infra) + - name: Start Kubernetes cluster run: > nix run .#native up - - name: Start Kubernetes cluster (Operations) + - name: Start NativeLink operator run: > nix develop --impure --command - bash -c "./deployment-examples/kubernetes/01_operations.sh" + bash -c "kubectl apply -k deployment-examples/kubernetes" - - name: Start Kubernetes cluster (Application) + - name: Wait for Tekton pipelines run: > nix develop --impure --command - bash -c "./deployment-examples/kubernetes/02_application.sh" + bash -c "kubectl wait \ + --for=condition=Succeeded \ + --timeout=45m \ + pipelinerun \ + -l tekton.dev/pipeline=rebuild-nativelink" + + - name: Wait for CAS + run: > + nix develop --impure --command + bash -c "kubectl rollout status deploy/nativelink-cas" + + - name: Wait for scheduler + run: > + nix develop --impure --command + bash -c "kubectl rollout status deploy/nativelink-scheduler" + + - name: Wait for worker + run: > + nix develop --impure --command + bash -c "kubectl rollout status deploy/nativelink-worker" - name: Get gateway IPs id: gateway-ips diff --git a/deployment-examples/chromium/03_build_chrome_tests.sh b/deploy/chromium-example/build_chromium_tests.sh similarity index 100% rename from deployment-examples/chromium/03_build_chrome_tests.sh rename to deploy/chromium-example/build_chromium_tests.sh diff --git a/deploy/chromium-example/kustomization.yaml b/deploy/chromium-example/kustomization.yaml new file mode 100644 index 0000000000..a6f329d510 --- /dev/null +++ b/deploy/chromium-example/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../kubernetes/components/operator + +patches: +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/overlays/chromium + target: + kind: Kustomization + name: nativelink +- patch: |- + - op: replace + path: /spec/url + value: https://github.com/aaronmondal/nativelink + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/ref/branch + value: flux + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium + target: + kind: Alert + name: nativelink-worker-alert diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml new file mode 100644 index 0000000000..acc552ec93 --- /dev/null +++ b/deploy/dev/kustomization.yaml @@ -0,0 +1,69 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../kubernetes/components/operator + +# Change this value to deploy custom overlays. +patches: +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/overlays/lre + target: + kind: Kustomization + name: nativelink + +# Modify this value to change the URL of the repository with deployment files. +# +# This is usually only necessary if you change deployment YAML files or +# NativeLink config files. If you only intend to change the Rust sources you can +# leave this as is and need to ensure that the Alerts below are patched to build +# your local sources. +- patch: |- + - op: replace + path: /spec/url + value: https://github.com/TraceMachina/nativelink + target: + kind: GitRepository + name: nativelink + +# Patch this to change to track a custom branch. +- patch: |- + - op: replace + path: /spec/ref/branch + value: main + target: + kind: GitRepository + name: nativelink + +# Setting the flake outputs to `./src_root#xxx` causes the Tekton pipelines to +# build nativelink from your local sources. +# +# During development, the following formats might be useful as well: +# +# `github:user/repo#outname` to build an image from an arbitrary flake output. +# +# `github:TraceMachina/nativelink?ref=pull//head#` to deploy a +# outputs from a Pull request. +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: ./src_root#image + target: + kind: Alert + name: nativelink-image-alert +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: ./src_root#nativelink-worker-init + target: + kind: Alert + name: nativelink-worker-init-alert +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: ./src_root#nativelink-worker-lre-cc + target: + kind: Alert + name: nativelink-worker-alert diff --git a/deploy/kubernetes-example/kustomization.yaml b/deploy/kubernetes-example/kustomization.yaml new file mode 100644 index 0000000000..89fb0b7989 --- /dev/null +++ b/deploy/kubernetes-example/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- ../../kubernetes/components/operator + +patches: +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/overlays/lre + target: + kind: Kustomization + name: nativelink +- patch: |- + - op: replace + path: /spec/url + value: https://github.com/aaronmondal/nativelink + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/ref/branch + value: flux + target: + kind: GitRepository + name: nativelink +- patch: |- + - op: replace + path: /spec/eventMetadata/flakeOutput + value: github:TraceMachina/nativelink#nativelink-worker-lre-cc + target: + kind: Alert + name: nativelink-worker-alert diff --git a/deployment-examples/chromium/.gitignore b/deployment-examples/chromium/.gitignore deleted file mode 100644 index e0e8ebd5ba..0000000000 --- a/deployment-examples/chromium/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Generated by 02_application.yaml -/kustomization.yaml diff --git a/deployment-examples/chromium/01_operations.sh b/deployment-examples/chromium/01_operations.sh deleted file mode 100755 index cd57c146f4..0000000000 --- a/deployment-examples/chromium/01_operations.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Trigger cluster-internal pipelines to build or fetch necessary images. - -set -xeuo pipefail - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#image"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-init"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-siso-chromium"}' \ - localhost:8082/eventlistener - -until kubectl get pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink | grep -q 'NAME'; do - echo "Waiting for PipelineRuns to start..." - sleep 0.1 -done - -printf "Waiting for PipelineRuns to finish... - -You may cancel this script now and use 'tkn pr ls' and 'tkn pr logs -f' to -monitor the PipelineRun logs. - -" - -kubectl wait \ - --for=condition=Succeeded \ - --timeout=45m \ - pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink diff --git a/deployment-examples/chromium/02_application.sh b/deployment-examples/chromium/02_application.sh deleted file mode 100755 index 6d1e717e67..0000000000 --- a/deployment-examples/chromium/02_application.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Prepare the Kustomization and apply it to the cluster. - -KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/chromium - -cat < "$KUSTOMIZE_DIR"/kustomization.yaml ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: - - ../kubernetes/base - -resources: - - worker-chromium.yaml -EOF - -cd "$KUSTOMIZE_DIR" && kustomize edit set image \ - nativelink=localhost:5001/nativelink:"$(\ - nix eval .#image.imageTag --raw)" \ - nativelink-worker-init=localhost:5001/nativelink-worker-init:"$(\ - nix eval .#nativelink-worker-init.imageTag --raw)" \ - nativelink-worker-chromium=localhost:5001/nativelink-worker-siso-chromium:"$(\ - nix eval .#nativelink-worker-siso-chromium.imageTag --raw)" - -kubectl apply -k "$KUSTOMIZE_DIR" - -kubectl rollout status deploy/nativelink-cas -kubectl rollout status deploy/nativelink-scheduler -kubectl rollout status deploy/nativelink-worker-chromium diff --git a/deployment-examples/chromium/04_delete_application.sh b/deployment-examples/chromium/04_delete_application.sh deleted file mode 100755 index 681370f921..0000000000 --- a/deployment-examples/chromium/04_delete_application.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Delete the Kustomization but leave the rest of the cluster intact. - -kubectl delete -k \ - "$(git rev-parse --show-toplevel)/deployment-examples/chromium" diff --git a/deployment-examples/chromium/README.md b/deployment-examples/chromium/README.md deleted file mode 100644 index bdadb379c6..0000000000 --- a/deployment-examples/chromium/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Chromium example - -This deployment sets up a 4-container deployment with separate CAS, scheduler -and worker. Don't use this example deployment in production. It's insecure. - -> [!WARNING] -> - The client build request is best done from a Ubuntu image, `./03_build_chrome_tests.sh`. It will check if the image is Ubuntu and -> fail otherwise. -> - This tutorial has been tested in a Nix environment of version `2. -> 21.0`. -> - You need to install the [Docker](https://docs.docker.com/engine/install/ubuntu/) Engine in Ubuntu. -> - To get your Nix environment set up see the [official Nix installation documentation](https://nix.dev/install-nix). - -All commands should be run from nix to ensure all dependencies exist in the environment. - -```bash -nix develop -``` - -In this example we're using `kind` to set up the cluster `cilium` to provide a -`LoadBalancer` and `GatewayController`. - -First set up a local development cluster: - -```bash -native up -``` - -> [!TIP] -> The `native up` command uses Pulumi under the hood. You can view and delete -> the stack with `pulumi stack` and `pulumi destroy`. - -Next start a few standard deployments. This part also builds the remote -execution containers and makes them available to the cluster: - -```bash -./01_operations.sh -``` - -> [!TIP] -> The operations invoke cluster-internal Tekton Pipelines to build and push the -> `nativelink` and worker images. You can view the state of the pipelines with -> `tkn pr ls` and `tkn pr logs`/`tkn pr logs --follow`. - -Finally, deploy NativeLink: - -```bash -./02_application.sh -``` - -> [!TIP] -> You can use `./04_delete_application.sh` to remove just the `nativelink` -> deployments but leave the rest of the cluster intact. - -This demo setup creates two gateways to expose the `cas` and `scheduler` -deployments via your local docker network: - -```bash -CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') -SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') - -echo "Cache IP: $CACHE" -echo "Scheduler IP: $SCHEDULER" -``` - -Using `./03_build_chrome_tests.sh` example script will download needed dependencies -for building Chromium unit tests using NativeLink CAS and Scheduler. The initial part -of the script checks if some dependencies exist, if not installs them, then moves on -to downloading and building Chromium tests. The script simplifies the setup described -in [linux/build_instructions.md](https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md) - -```bash -./03_build_chrome_tests.sh -``` - -> [!TIP] -> You can monitor the logs of container groups with `kubectl logs`: -> ```bash -> kubectl logs -f -l app=nativelink-cas -> kubectl logs -f -l app=nativelink-scheduler -> kubectl logs -f -l app=nativelink-worker-chromium --all-containers=true -> watch $HOME/chromium/src/buildtools/reclient/reproxystatus -> ``` - -When you're done testing, delete the cluster: - -```bash -kind delete cluster -``` -## NativeLink Community -If you have any questions, please reach out to the [NativeLink Community](https://join.slack.com/t/nativelink/shared_invite/zt-2i2mipfr5-lZAEeWYEy4Eru94b3IOcdg). diff --git a/deployment-examples/kubernetes/.gitignore b/deployment-examples/kubernetes/.gitignore deleted file mode 100644 index e0e8ebd5ba..0000000000 --- a/deployment-examples/kubernetes/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Generated by 02_application.yaml -/kustomization.yaml diff --git a/deployment-examples/kubernetes/01_operations.sh b/deployment-examples/kubernetes/01_operations.sh deleted file mode 100755 index 6265ea11d0..0000000000 --- a/deployment-examples/kubernetes/01_operations.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Trigger cluster-internal pipelines to build or fetch necessary images. - -set -xeuo pipefail - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#image"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-init"}' \ - localhost:8082/eventlistener - -curl -v \ - -H 'content-Type: application/json' \ - -d '{"flakeOutput": "./src_root#nativelink-worker-lre-cc"}' \ - localhost:8082/eventlistener - -until kubectl get pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink | grep -q 'NAME'; do - echo "Waiting for PipelineRuns to start..." - sleep 0.1 -done - -printf "Waiting for PipelineRuns to finish... - -You may cancel this script now and use 'tkn pr ls' and 'tkn pr logs -f' to -monitor the PipelineRun logs. - -" - -kubectl wait \ - --for=condition=Succeeded \ - --timeout=45m \ - pipelinerun \ - -l tekton.dev/pipeline=rebuild-nativelink diff --git a/deployment-examples/kubernetes/02_application.sh b/deployment-examples/kubernetes/02_application.sh deleted file mode 100755 index b444cf9794..0000000000 --- a/deployment-examples/kubernetes/02_application.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Prepare the Kustomization and apply it to the cluster. - -KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/kubernetes - -cat < "$KUSTOMIZE_DIR"/kustomization.yaml ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: - - base - -resources: - - worker-lre-cc.yaml - # TODO(aaronmondal): Fix java and add this: - # - worker-lre-java.yaml -EOF - -cd "$KUSTOMIZE_DIR" && kustomize edit set image \ - nativelink=localhost:5001/nativelink:"$(\ - nix eval .#image.imageTag --raw)" \ - nativelink-worker-init=localhost:5001/nativelink-worker-init:"$(\ - nix eval .#nativelink-worker-init.imageTag --raw)" \ - nativelink-worker-lre-cc=localhost:5001/nativelink-worker-lre-cc:"$(\ - nix eval .#nativelink-worker-lre-cc.imageTag --raw)" - -# TODO(aaronmondal): Fix java and add this: -# nativelink-worker-lre-java=localhost:5001/nativelink-worker-lre-java:$(\ -# nix eval .#nativelink-worker-lre-java.imageTag --raw) - -kubectl apply -k "$KUSTOMIZE_DIR" - -kubectl rollout status deploy/nativelink-cas -kubectl rollout status deploy/nativelink-scheduler -kubectl rollout status deploy/nativelink-worker-lre-cc - -# TODO(aaronmondal): Fix java and add this: -# kubectl rollout status deploy/nativelink-worker-lre-java diff --git a/deployment-examples/kubernetes/03_delete_application.sh b/deployment-examples/kubernetes/03_delete_application.sh deleted file mode 100755 index c26119e863..0000000000 --- a/deployment-examples/kubernetes/03_delete_application.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Delete the Kustomization but leave the rest of the cluster intact. - -kubectl delete -k \ - "$(git rev-parse --show-toplevel)/deployment-examples/kubernetes" diff --git a/deployment-examples/kubernetes/README.md b/deployment-examples/kubernetes/README.md deleted file mode 100644 index 9fdc8510b1..0000000000 --- a/deployment-examples/kubernetes/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# Kubernetes example - -This deployment sets up a 4-container deployment with separate CAS, scheduler -and worker. Don't use this example deployment in production. It's insecure. - -In this example we're using `kind` to set up the cluster `cilium` to provide a -`LoadBalancer` and `GatewayController`. - -First set up a local development cluster: - -```bash -native up -``` - -> [!TIP] -> The `native up` command uses Pulumi under the hood. You can view and delete -> the stack with `pulumi stack` and `pulumi destroy`. - -Next start a few standard deployments. This part also builds the remote -execution containers and makes them available to the cluster: - -```bash -./01_operations.sh -``` - -> [!TIP] -> The operations invoke cluster-internal Tekton Pipelines to build and push the -> `nativelink` and worker images. You can view the state of the pipelines with -> `tkn pr ls` and `tkn pr logs`/`tkn pr logs --follow`. - -Finally, deploy NativeLink: - -```bash -./02_application.sh -``` - -> [!TIP] -> You can use `./03_delete_application.sh` to remove just the `nativelink` -> deployments but leave the rest of the cluster intact. - -This demo setup creates two gateways to expose the `cas` and `scheduler` -deployments via your local docker network: - -```bash -CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') -SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') - -echo "Cache IP: $CACHE" -echo "Scheduler IP: $SCHEDULER" - -# Prints something like: -# -# Cache IP: 172.20.255.4 -# Scheduler IP: 172.20.255.5 -``` - -You can now pass these IP addresses to your Bazel invocation to use the remote -cache and executor: - -```bash -bazel build \ - --config=lre \ - --remote_instance_name=main \ - --remote_cache=grpc://$CACHE \ - --remote_executor=grpc://$SCHEDULER \ - //local-remote-execution/examples:hello_lre -``` - -> [!TIP] -> You can add these flags to a to a `.bazelrc.user` file in the workspace root. -> Note that you'll need to pass in explicit IP addresses as this file can't -> resolve environment variables: -> ```bash -> # .bazelrc.user -> build --config=lre -> build --remote_instance_name=main -> build --remote_cache=grpc://172.20.255.4 -> build --remote_executor=grpc://172.20.255.5 -> ``` - -When you're done testing, delete the cluster: - -```bash -kind delete cluster -``` - -## Use a published image - -[Published images](https://github.com/TraceMachina/nativelink/pkgs/container/nativelink) can be found under the Container registry, which uses the namespace `https://ghcr.io`. When using the Container registry, you can select prebuilt images and avoid building the image yourself. - -To pull an existing image, you can run: - -```sh -docker pull ghcr.io/tracemachina/nativelink:taggedImageVersion -``` - -## Derive a Tag for an OCI image - -To derive the tag of the NativeLink image at a specific commit, run the below command and change `someCommit` with the commit hash you want to use: - -```sh -nix eval github:TraceMachina/nativelink/someCommit#image.imageTag --raw -``` - -Alternatively, the tag can be derived from the upstream sources at the current state of the upstream main branch by running this command: - -```sh -nix eval github:TraceMachina/nativelink#image.imageTag --raw -``` - -Similarly, you can also clone or checkout a specific version or commit of the NativeLink git repository to evaluate the output of the entire NativeLink flake. For example, assuming you've done the [NativeLink Getting Started Guide](https://github.com/TraceMachina/nativelink?tab=readme-ov-file#getting-started-with-nativelink) and cloned the repository, you can run these sample commands: - -```sh -git log -git checkout commitHash -nix eval .#image.imageTag --raw -``` -The `--raw` removes the surrounding quotes from the output string. - -> [!WARNING] -> We don't recommend using this command to -> retrieve an image: -> ```sh -> nix eval github:TraceMachina/nativelink#image.imageTag --raw -> ``` -> Using this command prevents anyone from -> identifying the specific version of the -> NativeLink container in use because -> reflects the image version available at the -> time of download. It'll be hard to debug, -> revert to previous versions if there are issues -> and complicate bug tracking. -> It's for these same reasons you won't be able -> to retrieve an image using the `latest` tag. - -## Build and copy an OCI image - -You can build and copy the image to a container registry with the `copyTo` attribute. Below are examples within the NativeLink repository for building and copying an image: -- [Example 1](https://github.com/TraceMachina/nativelink/blob/09b32c94d3cc7780816585e9b87f69c56cf931ae/deployment-examples/kubernetes/01_operations.sh#L12-L16) highlights: - -```sh -nix run github:tracemachina/nativelink#image.copyTo -``` - -- [Example 2](https://github.com/TraceMachina/nativelink/blob/09b32c94d3cc7780816585e9b87f69c56cf931ae/tools/local-image-test.nix#L12-L13) highlights how to skip pushing to an intermediary registry by copying directly to the docker-daemon: - -```sh -IMAGE_NAME=$(nix eval .#image.imageName --raw) -IMAGE_TAG=$(nix eval .#image.imageTag --raw) - -nix run .#image.copyTo docker-daemon:"${IMAGE_NAME}":"${IMAGE_TAG}" -``` - -You can find more about details around [nix](https://github.com/nlewo/nix2container). Published images are signed using `cosign`. For more details of the verification process of publishing OCI images see [SECURITY.md](https://github.com/TraceMachina/nativelink/blob/main/SECURITY.md) - -## NativeLink Community - -Reach out to the [NativeLink Slack community](https://join.slack.com/t/nativelink/shared_invite/zt-2forhp5n9-L7dTD21nCSY9_IRteQvZmw) for any questions via #NativeLink! diff --git a/deployment-examples/kubernetes/base/kustomization.yaml b/deployment-examples/kubernetes/base/kustomization.yaml deleted file mode 100644 index 8bc3a9d247..0000000000 --- a/deployment-examples/kubernetes/base/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - cas.yaml - - scheduler.yaml - - routes.yaml - -configMapGenerator: - - name: cas - files: - - cas.json - - name: scheduler - files: - - scheduler.json - - name: worker - files: - - worker.json - -images: - - name: nativelink - -secretGenerator: - - name: tls-secret - files: - - example-do-not-use-in-prod-rootca.crt - - example-do-not-use-in-prod-key.pem diff --git a/deployment-examples/kubernetes/worker-lre-java.yaml b/deployment-examples/kubernetes/worker-lre-java.yaml deleted file mode 100644 index d4c5804810..0000000000 --- a/deployment-examples/kubernetes/worker-lre-java.yaml +++ /dev/null @@ -1,69 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nativelink-worker-lre-java -spec: - replicas: 1 - selector: - matchLabels: - app: nativelink-worker-lre-java - template: - metadata: - labels: - app: nativelink-worker-lre-java - spec: - initContainers: - - name: setup-entrypoint - image: nixpkgs/nix-flakes:latest - command: ["/bin/sh", "-c"] - # The kind setup mounts the nativelink repository into the kind nodes at - # `/mnt/src_root`. This ensures that the tags between the worker configs - # and bazel toolchains match when this setup is run in CI. - # - # WARNING: The platform is *not* necessarily the container that is - # actually deployed here. The generator container in this example was - # `rbe-autogen-lre-java:` and the platform was modified - # after the fact to be `lre-java:`. The deployed container - # we use as worker is - # `nativelink-worker-lre-java:` which is a - # completely separate extension of the `lre-java` base image. - args: - - | - git config --global --add safe.directory "*" - NATIVELINK_WORKER_PLATFORM=docker://lre-java:$(nix eval /mnt/src_root#lre-java.imageTag --raw) && - printf '#!/bin/sh\nexport NATIVELINK_WORKER_PLATFORM=%s\nexec "$@"' "$NATIVELINK_WORKER_PLATFORM" > /entrypoint/entrypoint.sh && - chmod +x /entrypoint/entrypoint.sh - volumeMounts: - - name: entrypoint - mountPath: /entrypoint - - name: mnt - mountPath: /mnt - containers: - - name: nativelink-worker-lre-java - # This image will be edited by kustomize - image: nativelink-worker-lre-java - env: - - name: RUST_LOG - value: warn - - name: CAS_ENDPOINT - value: nativelink-cas - - name: SCHEDULER_ENDPOINT - value: nativelink-scheduler - volumeMounts: - - name: worker-config - mountPath: /worker.json - subPath: worker.json - - name: entrypoint - mountPath: /entrypoint - command: ["/entrypoint/entrypoint.sh"] - args: ["/bin/nativelink", "/worker.json"] - volumes: - - name: entrypoint - emptyDir: {} - - name: worker-config - configMap: - name: worker - - name: mnt - hostPath: - path: /mnt diff --git a/docs/.gitignore b/docs/.gitignore index 1c24988f18..abc47868d0 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -32,7 +32,5 @@ src/content/docs/explanations/lre.mdx src/content/docs/reference/changelog.mdx src/content/docs/reference/nativelink-config.mdx src/content/docs/tutorials/setup.mdx -src/content/docs/guides/chromium.mdx src/content/docs/guides/configuration.mdx -src/content/docs/guides/kubernetes.mdx src/content/docs/guides/setup.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index faf7cc850e..4c4760bf7a 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -106,16 +106,16 @@ export default defineConfig({ label: "Guides", items: [ { - label: "Configuration examples", - link: "/guides/configuration", + label: "Kubernetes example", + link: "/guides/kubernetes", }, { label: "Chromium example", link: "/guides/chromium", }, { - label: "Kubernetes example", - link: "/guides/kubernetes", + label: "Configuration examples", + link: "/guides/configuration", }, { label: "Contributing", diff --git a/docs/scripts/md_to_mdx_aot.ts b/docs/scripts/md_to_mdx_aot.ts index 613091d81e..e5f23dbce9 100644 --- a/docs/scripts/md_to_mdx_aot.ts +++ b/docs/scripts/md_to_mdx_aot.ts @@ -58,16 +58,6 @@ convertMarkdownToMdx( "src/content/docs/guides/configuration.mdx", "NativeLink configuration guide", ); -convertMarkdownToMdx( - "../deployment-examples/chromium/README.md", - "src/content/docs/guides/chromium.mdx", - "NativeLink deployment example for Chromium", -); -convertMarkdownToMdx( - "../deployment-examples/kubernetes/README.md", - "src/content/docs/guides/kubernetes.mdx", - "NativeLink deployment example for Kubernetes", -); convertMarkdownToMdx( "../CHANGELOG.md", "src/content/docs/reference/changelog.mdx", diff --git a/docs/src/content/docs/guides/chromium.mdx b/docs/src/content/docs/guides/chromium.mdx new file mode 100644 index 0000000000..9935353b4e --- /dev/null +++ b/docs/src/content/docs/guides/chromium.mdx @@ -0,0 +1,120 @@ +--- +title: Chromium example +description: 'An example for building Chromium with NativeLink in Kubernetes.' +--- + +In this example you'll spin up a local Kubernetes cluster with NativeLink and +run a Chromium build against it. + +**Requirements** + +- An `x86_64-linux` system running a recent Ubuntu. Either "real" Linux or WSL2. +- A functional local Docker setup. +- A recent version of Nix with flake support, for instance installed via the + [nextgen Nix installer](https://github.com/NixOS/experimental-nix-installer). + +:::caution +This example doesn't work on Mac and likely doesn't work on Linux distributions +other than Ubuntu. +::: + +## โ˜๏ธ Prepare the cluster + +First, enter the NativeLink development environment: + +```bash +# TODO(aaronmondal): Point to the main repo before merging. +git clone https://github.com/aaronmondal/nativelink && \ + cd nativelink && \ + git switch flux \ + nix develop +``` + +This environment contains some cloud tooling, so you don't need to set up any +kubernetes-related software yourself. + +Now, start the development cluster: + +```bash +native up +``` + +:::tip +The `native up` command uses Pulumi under the hood. You can view and delete +the stack with `pulumi stack` and `pulumi destroy`. If you're queried for a +stack password, press enter, as the password is an empty string. +::: + +Next, deploy NativeLink to the cluster: + +```bash +kubectl apply -k \ + https://github.com/aaronmondal/nativelink//deploy/chromium-example?ref=flux +``` + +:::danger +This example is built for demo purposes only. It's not a secure production-grade +setup and will only work in the local devcluster created with `native up`. + +One-liner production-grade setups are still under construction. +::: + +## ๐Ÿ”ญ Explore deployments + +The deployment might take a wile to boot up. You can monitor progress via the +dashboards that come with the development cluster: + +- [localhost:8080](http://localhost:8080): Cilium's Hubble UI to view the + cluster topology. NativeLink will be deployed into the `default` namespace. +- [localhost:8081](http://localhost:8081): The Tekton Dashboard to view the + progress of the in-cluster pipelines. You'll find the pipelines under the + `PipelineRuns` tab. +- [localhost:9000](http://localhost:9000): The Capacitor Dashboard to view Flux + Kustomizations. You can view NatieLink's logs here once it's fully deployed. + +In terminals, the following comands can be helpful to view deployment progress: + +- `tkn pr logs -f` to view the logs of a pipelinerun in the terminal. +- `flux get all -A` to view the state of the NativeLink deployments. +- Once NativeLink is deployed: + - `kubectl logs deploy/nativelink-cas` for the CAS (cache) logs. + - `kubectl logs deploy/nativelink-scheduler` for the scheduler logs. + - `kubectl logs deploy/nativelink-worker` for the worker logs. + +## ๐Ÿ—๏ธ Build against NativeLink + +The demo setup creates gateways to expose the `cas` and `scheduler` deployments +via your local docker network. The following command builds the chromium tests +against the cluster: + +```bash +build-chromium-tests +``` + +The `build-chromium-tests` command simplifies the setup described in +[linux/build_instructions.md](https://chromium.googlesource.com/chromium/src/+/main/docs/linux/build_instructions.md). +After preparing the requirements, it runs a Reclient build against the cluster. + +:::note +See [deploy/chromium-example/build_chromium_tests.sh](https://github.com/TraceMachina/nativelink/blob/main/deploy/chromium-example/build_chromium_tests.sh) +for the script contents. +::: + +You can view Reclient's logs like so: + +```bash +watch $HOME/chromium/src/buildtools/reclient/reproxystatus +``` + +## ๐Ÿงน Clean up + +When you're done testing, delete the cluster: + +```bash +# Delete the kind cluster +native down + +# Remove the container registry and loadbalancer +docker container stop kind-registry | xargs docker rm +docker container stop kind-loadbalancer | xargs docker rm +``` diff --git a/docs/src/content/docs/guides/kubernetes.mdx b/docs/src/content/docs/guides/kubernetes.mdx new file mode 100644 index 0000000000..69d551b37b --- /dev/null +++ b/docs/src/content/docs/guides/kubernetes.mdx @@ -0,0 +1,190 @@ +--- +title: Kubernetes example +description: 'An example setup for NativeLink in Kubernetes' +--- + +In this example you'll spin up a local Kubernetes cluster with NativeLink and +run some Bazel builds against it. + +**Requirements** + +- An `x86_64-linux` system. Either "real" Linux or WSL2. +- A functional local Docker setup. +- A recent version of Nix with flake support, for instance installed via the + [nextgen Nix installer](https://github.com/NixOS/experimental-nix-installer). + +:::caution +This example doesn't work on Mac yet. +::: + + +## โ˜๏ธ Prepare the cluster + +First, enter the NativeLink development environment: + +```bash +# TODO(aaronmondal): Point to the main repo before merging. +git clone https://github.com/aaronmondal/nativelink && \ + cd nativelink && \ + git switch flux \ + nix develop +``` + +This environment contains Bazel and some cloud tooling, so you don't need to set +up any kubernetes-related software yourself. + +Now, start the development cluster: + +```bash +native up +``` + +:::tip +The `native up` command uses Pulumi under the hood. You can view and delete +the stack with `pulumi stack` and `pulumi destroy`. If you're queried for a +stack password, press enter, as the password is an empty string. +::: + +Next, deploy NativeLink to the cluster: + +```bash +kubectl apply -k \ + https://github.com/aaronmondal/nativelink//deploy/kubernetes-example?ref=flux +``` + +:::danger +This example is built for demo purposes only. It's not a secure production-grade +setup and will only work in the local devcluster created with `native up`. + +One-liner production-grade setups are still under construction. +::: + +## ๐Ÿ”ญ Explore deployments + +The deployment might take a wile to boot up. You can monitor progress via the +dashboards that come with the development cluster: + +- [localhost:8080](http://localhost:8080): Cilium's Hubble UI to view the + cluster topology. NativeLink will be deployed into the `default` namespace. +- [localhost:8081](http://localhost:8081): The Tekton Dashboard to view the + progress of the in-cluster pipelines. You'll find the pipelines under the + `PipelineRuns` tab. +- [localhost:9000](http://localhost:9000): The Capacitor Dashboard to view Flux + Kustomizations. You can view NatieLink's logs here once it's fully deployed. + +In terminals, the following comands can be helpful to view deployment progress: + +- `tkn pr logs -f` to view the logs of a pipelinerun in the terminal. +- `flux get all -A` to view the state of the NativeLink deployments. +- Once NativeLink is deployed: + - `kubectl logs deploy/nativelink-cas` for the CAS (cache) logs. + - `kubectl logs deploy/nativelink-scheduler` for the scheduler logs. + - `kubectl logs deploy/nativelink-worker` for the worker logs. + +## ๐Ÿ—๏ธ Build against NativeLink + +The demo setup creates gateways to expose the `cas` and `scheduler` deployments +via your local docker network. You can pass the Gateway addresses to Bazel +invocations to make builds run against the cluster: + +```bash +CACHE=$(kubectl get gtw cache-gateway -o=jsonpath='{.status.addresses[0].value}') +SCHEDULER=$(kubectl get gtw scheduler-gateway -o=jsonpath='{.status.addresses[0].value}') + +echo "Cache IP: $CACHE" +echo "Scheduler IP: $SCHEDULER" + +bazel build \ + --config=lre \ + --remote_instance_name=main \ + --remote_cache=grpc://$CACHE \ + --remote_executor=grpc://$SCHEDULER \ + //local-remote-execution/examples:hello_lre +``` + +:::caution +While the Dashboard ports are static, the NativeLink endpoints are not (yet). +If you shut down the cluster and reboot it, the `$CACHE` and `$SCHEDULER` IPs +will change. +::: + +:::tip +You can add these flags to a to a `.bazelrc.user` file in the workspace root. +Note that you'll need to pass in explicit IP addresses as this file can't +resolve environment variables: +```bash +# .bazelrc.user +build --config=lre +build --remote_instance_name=main +build --remote_cache=grpc://172.20.255.4 +build --remote_executor=grpc://172.20.255.5 +``` + +```bash +# .bazelrc +try-import %workspace%/.bazelrc.user +``` +::: + +The crucial part is this bit: + +```txt +INFO: 11 processes: 9 internal, 2 remote. +``` + +It tells us that the compilation ran against the cluster. Let's clean the Bazel +cache and run the build again: + +```bash +bazel clean && bazel build \ + --config=lre \ + --remote_instance_name=main \ + --remote_cache=grpc://$CACHE \ + --remote_executor=grpc://$SCHEDULER \ + //local-remote-execution/examples:hello_lre +``` + +The build now shows cache hits instead of remote actions: + +```txt +INFO: 11 processes: 2 remote cache hit, 9 internal. +``` + +## ๐Ÿš€ Bonus: Local Remote Execution + +The worker deployment in this example leverages [Local Remote Execution](../explanations/lre). + +Local Remote Execution mirrors toolchains for remote execution in your local +development environment. This lets you reuse build artifacts with virtually +perfect cache hit rate across different repositories, developers, and CI. + +To test LRE in the cluster, clean the local cache and invoke another build +against the cluster, but this time omit the `remote_executor` flag. This way +you'll use remote caching without remote execution: + +```bash +bazel clean && bazel build \ + --config=lre \ + --remote_instance_name=main \ + --remote_cache=grpc://$CACHE \ + //local-remote-execution/examples:hello_lre +``` + +You'll get remote cache hits as if your local machine was a `nativelink-worker`: + +```txt +INFO: 11 processes: 2 remote cache hit, 9 internal. +``` + +## ๐Ÿงน Clean up + +When you're done testing, delete the cluster: + +```bash +# Delete the kind cluster +native down + +# Remove the container registry and loadbalancer +docker container stop kind-registry | xargs docker rm +docker container stop kind-loadbalancer | xargs docker rm +``` diff --git a/flake.nix b/flake.nix index 8597e051d1..d69c422c83 100644 --- a/flake.nix +++ b/flake.nix @@ -145,6 +145,11 @@ native-cli = import ./native-cli/default.nix {inherit pkgs;}; + build-chromium-tests = + pkgs.writeShellScriptBin + "build-chromium-tests" + ./deploy/chromium-example/build_chromium_tests.sh; + docs = pkgs.callPackage ./tools/docs.nix {rust = stable-rust.default;}; inherit (nix2container.packages.${system}.nix2container) pullImage; @@ -283,6 +288,7 @@ pkgs.docker-client pkgs.kind pkgs.tektoncd-cli + pkgs.fluxcd (pkgs.pulumi.withPackages (ps: [ps.pulumi-language-go])) pkgs.go pkgs.kustomize @@ -294,6 +300,7 @@ customClang native-cli docs + build-chromium-tests ] ++ pkgs.lib.optionals (!pkgs.stdenv.isDarwin) [ # The docs on Mac require a manual setup outside the flake. diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 0000000000..dd516969af --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,13 @@ +# NativeLink Kubernetes deployments + +Building blocks for NativeLink Kubernetes deployments. + +This directory does **not** contain a one-size-fits-all solution like a Helm +chart - infrastructure requirements are too diverse for a single setup to +reliably cover all potential use-cases. + +Instead, we provide useful building blocks in the form of Kustomizations. +Downstream implementers might use them as reference points to patch in the +functionality they require. + +See the `deployment-examples` directory for concrete example deployments. diff --git a/deployment-examples/kubernetes/base/cas.yaml b/kubernetes/base/cas.yaml similarity index 100% rename from deployment-examples/kubernetes/base/cas.yaml rename to kubernetes/base/cas.yaml diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml new file mode 100644 index 0000000000..92d4e29f7d --- /dev/null +++ b/kubernetes/base/kustomization.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cas.yaml + - scheduler.yaml + - worker.yaml + - ../configmaps + +images: + - name: nativelink + - name: nativelink-worker-init + - name: nativelink-worker diff --git a/deployment-examples/kubernetes/base/scheduler.yaml b/kubernetes/base/scheduler.yaml similarity index 100% rename from deployment-examples/kubernetes/base/scheduler.yaml rename to kubernetes/base/scheduler.yaml diff --git a/deployment-examples/chromium/worker-chromium.yaml b/kubernetes/base/worker.yaml similarity index 75% rename from deployment-examples/chromium/worker-chromium.yaml rename to kubernetes/base/worker.yaml index 29ca10a7ff..dcf57bc2c4 100644 --- a/deployment-examples/chromium/worker-chromium.yaml +++ b/kubernetes/base/worker.yaml @@ -2,16 +2,16 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-worker-chromium + name: nativelink-worker spec: replicas: 3 selector: matchLabels: - app: nativelink-worker-chromium + app: nativelink-worker template: metadata: labels: - app: nativelink-worker-chromium + app: nativelink-worker spec: initContainers: - name: nativelink-worker-init @@ -21,11 +21,10 @@ spec: volumeMounts: - name: shared mountPath: /shared - containers: - - name: nativelink-worker-chromium + - name: nativelink-worker # This image will be edited by kustomize. - image: nativelink-worker-chromium + image: nativelink-worker env: - name: RUST_LOG value: info @@ -37,6 +36,8 @@ spec: - name: worker-config mountPath: /worker.json subPath: worker.json + - name: shared + mountPath: /shared command: ["/shared/nativelink"] args: ["/worker.json"] volumes: @@ -45,3 +46,12 @@ spec: - name: worker-config configMap: name: worker +--- +apiVersion: v1 +kind: Service +metadata: + name: nativelink-worker +spec: + selector: + app: nativelink-worker + clusterIP: None diff --git a/kubernetes/components/gateway-routes/kustomization.yaml b/kubernetes/components/gateway-routes/kustomization.yaml new file mode 100644 index 0000000000..42b112b7e6 --- /dev/null +++ b/kubernetes/components/gateway-routes/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - routes.yaml diff --git a/deployment-examples/kubernetes/base/routes.yaml b/kubernetes/components/gateway-routes/routes.yaml similarity index 100% rename from deployment-examples/kubernetes/base/routes.yaml rename to kubernetes/components/gateway-routes/routes.yaml diff --git a/deployment-examples/kubernetes/base/example-do-not-use-in-prod-key.pem b/kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem similarity index 100% rename from deployment-examples/kubernetes/base/example-do-not-use-in-prod-key.pem rename to kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem diff --git a/deployment-examples/kubernetes/base/example-do-not-use-in-prod-rootca.crt b/kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt similarity index 100% rename from deployment-examples/kubernetes/base/example-do-not-use-in-prod-rootca.crt rename to kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt diff --git a/kubernetes/components/insecure-certs/kustomization.yaml b/kubernetes/components/insecure-certs/kustomization.yaml new file mode 100644 index 0000000000..bc92cb026f --- /dev/null +++ b/kubernetes/components/insecure-certs/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +secretGenerator: + - name: tls-secret + files: + - example-do-not-use-in-prod-rootca.crt + - example-do-not-use-in-prod-key.pem diff --git a/kubernetes/components/operator/flux-config.yaml b/kubernetes/components/operator/flux-config.yaml new file mode 100644 index 0000000000..c016867f52 --- /dev/null +++ b/kubernetes/components/operator/flux-config.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: nativelink-webhook + namespace: flux-system +spec: + type: generic + address: http://el-nativelink-rebuild.default.svc.cluster.local:8080 +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-image-alert + namespace: flux-system +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: github:TraceMachina/nativelink#image +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-worker-init-alert + namespace: flux-system +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: github:TraceMachina/nativelink#nativelink-worker-init +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-worker-alert + namespace: flux-system +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: "PLACEHOLDER_NATIVELINK_WORKER" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink-configmaps + namespace: default +spec: + interval: 2m + path: "./kubernetes/configmaps" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + path: "PLACEHOLDER_NATIVELINK_KUSTOMIZATION" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + postBuild: + substituteFrom: + - kind: ConfigMap + name: nativelink-image-tags + dependsOn: + - name: nativelink-configmaps diff --git a/kubernetes/components/operator/kustomization.yaml b/kubernetes/components/operator/kustomization.yaml new file mode 100644 index 0000000000..916d215e1e --- /dev/null +++ b/kubernetes/components/operator/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: +- flux-config.yaml diff --git a/deployment-examples/kubernetes/base/cas.json b/kubernetes/configmaps/cas.json similarity index 100% rename from deployment-examples/kubernetes/base/cas.json rename to kubernetes/configmaps/cas.json diff --git a/kubernetes/configmaps/kustomization.yaml b/kubernetes/configmaps/kustomization.yaml new file mode 100644 index 0000000000..91752af244 --- /dev/null +++ b/kubernetes/configmaps/kustomization.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: + - name: cas + files: + - cas.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + - name: scheduler + files: + - scheduler.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + - name: worker + files: + - worker.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled diff --git a/deployment-examples/kubernetes/base/scheduler.json b/kubernetes/configmaps/scheduler.json similarity index 100% rename from deployment-examples/kubernetes/base/scheduler.json rename to kubernetes/configmaps/scheduler.json diff --git a/deployment-examples/kubernetes/base/worker.json b/kubernetes/configmaps/worker.json similarity index 100% rename from deployment-examples/kubernetes/base/worker.json rename to kubernetes/configmaps/worker.json diff --git a/kubernetes/overlays/chromium/kustomization.yaml b/kubernetes/overlays/chromium/kustomization.yaml new file mode 100644 index 0000000000..10837536be --- /dev/null +++ b/kubernetes/overlays/chromium/kustomization.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base + +components: +- ../../components/gateway-routes +- ../../components/insecure-certs + +images: + - name: nativelink + newName: localhost:5001/nativelink + newTag: ${NATIVELINK_TAG} + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker + newName: localhost:5001/nativelink-worker-siso-chromium + newTag: ${NATIVELINK_WORKER_SISO_CHROMIUM_TAG} diff --git a/kubernetes/overlays/lre/kustomization.yaml b/kubernetes/overlays/lre/kustomization.yaml new file mode 100644 index 0000000000..707804b4ef --- /dev/null +++ b/kubernetes/overlays/lre/kustomization.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base + +components: +- ../../components/gateway-routes +- ../../components/insecure-certs + +patches: +- path: worker-lre-cc.yaml + target: + kind: Deployment + name: nativelink-worker + +images: + - name: nativelink + newName: localhost:5001/nativelink + newTag: ${NATIVELINK_TAG} + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker + newName: localhost:5001/nativelink-worker-lre-cc + newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} diff --git a/deployment-examples/kubernetes/worker-lre-cc.yaml b/kubernetes/overlays/lre/worker-lre-cc.yaml similarity index 63% rename from deployment-examples/kubernetes/worker-lre-cc.yaml rename to kubernetes/overlays/lre/worker-lre-cc.yaml index 36b79aed64..cbdb14c060 100644 --- a/deployment-examples/kubernetes/worker-lre-cc.yaml +++ b/kubernetes/overlays/lre/worker-lre-cc.yaml @@ -2,16 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-worker-lre-cc + name: nativelink-worker spec: replicas: 1 - selector: - matchLabels: - app: nativelink-worker-lre-cc template: - metadata: - labels: - app: nativelink-worker-lre-cc spec: initContainers: - name: setup-entrypoint @@ -39,44 +33,16 @@ spec: mountPath: /entrypoint - name: mnt mountPath: /mnt - - - name: nativelink-worker-init - # This image will be edited by kustomize. - image: nativelink-worker-init - args: ["/shared/nativelink"] - volumeMounts: - - name: shared - mountPath: /shared - containers: - - name: nativelink-worker-lre-cc - # This image will be edited by kustomize. - image: nativelink-worker-lre-cc - env: - - name: RUST_LOG - value: warn - - name: CAS_ENDPOINT - value: nativelink-cas - - name: SCHEDULER_ENDPOINT - value: nativelink-scheduler + - name: nativelink-worker volumeMounts: - - name: worker-config - mountPath: /worker.json - subPath: worker.json - name: entrypoint mountPath: /entrypoint - - name: shared - mountPath: /shared command: ["/entrypoint/entrypoint.sh"] args: ["/shared/nativelink", "/worker.json"] volumes: - - name: shared - emptyDir: {} - name: entrypoint emptyDir: {} - - name: worker-config - configMap: - name: worker - name: mnt hostPath: path: /mnt diff --git a/native-cli/clusters/localcluster.go b/native-cli/clusters/localcluster.go index dc7b253237..8b63ad47f9 100644 --- a/native-cli/clusters/localcluster.go +++ b/native-cli/clusters/localcluster.go @@ -10,7 +10,7 @@ import ( "runtime" "text/template" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" git "github.com/go-git/go-git/v5" "sigs.k8s.io/kind/pkg/cluster" @@ -210,7 +210,7 @@ func createRegistryConfigInNode( ) error { config := fmt.Sprintf("[host.\"http://%s:%d\"]", regName, internalPort) regDir := fmt.Sprintf("/etc/containerd/certs.d/localhost:%d", externalPort) - execConfig := types.ExecConfig{ + execConfig := container.ExecOptions{ Cmd: []string{ "sh", "-c", @@ -232,7 +232,7 @@ func createRegistryConfigInNode( ) } - if err := cli.ContainerExecStart(ctx, execID.ID, types.ExecStartCheck{}); err != nil { + if err := cli.ContainerExecStart(ctx, execID.ID, container.ExecAttachOptions{}); err != nil { return fmt.Errorf( "error starting exec command on node %s: %w", nodeName, diff --git a/native-cli/components/cilium.go b/native-cli/components/cilium.go index 31e61e1aaa..feb3b7bd79 100644 --- a/native-cli/components/cilium.go +++ b/native-cli/components/cilium.go @@ -8,7 +8,7 @@ import ( "slices" "strings" - "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" helmv3 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" @@ -134,7 +134,7 @@ func kindIPv4Subnet() (string, error) { return "", fmt.Errorf("%w: %w", errPulumi, err) } - networks, err := cli.NetworkList(dockerCtx, types.NetworkListOptions{}) + networks, err := cli.NetworkList(dockerCtx, network.ListOptions{}) if err != nil { return "", fmt.Errorf("%w: %w", errPulumi, err) } diff --git a/native-cli/components/embedded/capacitor.yaml b/native-cli/components/embedded/capacitor.yaml new file mode 100644 index 0000000000..e9de009148 --- /dev/null +++ b/native-cli/components/embedded/capacitor.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: capacitor + namespace: flux-system +spec: + interval: 12h + url: oci://ghcr.io/gimlet-io/capacitor-manifests + ref: + semver: ">=0.1.0" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: capacitor + namespace: flux-system +spec: + targetNamespace: flux-system + interval: 1h + retryInterval: 2m + timeout: 5m + wait: true + prune: true + path: "./" + sourceRef: + kind: OCIRepository + name: capacitor +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-ingress-to-capacitor + namespace: flux-system +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: onechart + app.kubernetes.io/instance: capacitor + ingress: + - fromEntities: + - ingress + - toPorts: + - ports: + - port: "9000" + protocol: TCP diff --git a/native-cli/components/embedded/envoy.template.yaml b/native-cli/components/embedded/envoy.template.yaml index 4dbe7051d3..4438265550 100644 --- a/native-cli/components/embedded/envoy.template.yaml +++ b/native-cli/components/embedded/envoy.template.yaml @@ -32,6 +32,8 @@ static_resources: {{- end }} http_filters: - name: envoy.filters.http.router + upgrade_configs: + - upgrade_type: websocket {{- end }} clusters: {{- range .InternalGateways }} diff --git a/native-cli/components/embedded/kustomization.yaml b/native-cli/components/embedded/kustomization.yaml index 945ec1d9ee..ee96870803 100644 --- a/native-cli/components/embedded/kustomization.yaml +++ b/native-cli/components/embedded/kustomization.yaml @@ -7,5 +7,7 @@ resources: - skopeo-check-hashlocked-url.yaml - nix2container-image-info.yaml - trigger.yaml + - update-image-tags.yaml + - capacitor.yaml # - nativelink-gateways.yaml # Gateways are handled in Pulumi via the # NativeLinkGateways resource. diff --git a/native-cli/components/embedded/nativelink-gateways.yaml b/native-cli/components/embedded/nativelink-gateways.yaml index a9ba34bf16..960289ad0a 100644 --- a/native-cli/components/embedded/nativelink-gateways.yaml +++ b/native-cli/components/embedded/nativelink-gateways.yaml @@ -57,3 +57,15 @@ spec: - name: tkn-gateway protocol: HTTP port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: capacitor-gateway + namespace: flux-system +spec: + gatewayClassName: cilium + listeners: + - name: capacitor-gateway + protocol: HTTP + port: 80 diff --git a/native-cli/components/embedded/nativelink-routes.yaml b/native-cli/components/embedded/nativelink-routes.yaml index 0dbc8e8e08..192b098edc 100644 --- a/native-cli/components/embedded/nativelink-routes.yaml +++ b/native-cli/components/embedded/nativelink-routes.yaml @@ -49,3 +49,20 @@ spec: backendRefs: - name: tekton-dashboard port: 9097 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: capacitor-route + namespace: flux-system +spec: + parentRefs: + - sectionName: capacitor-gateway + name: capacitor-gateway + rules: + - matches: + - path: + value: / + backendRefs: + - name: capacitor + port: 9000 diff --git a/native-cli/components/embedded/rebuild-nativelink.yaml b/native-cli/components/embedded/rebuild-nativelink.yaml index 4259377d77..eb540dd78a 100644 --- a/native-cli/components/embedded/rebuild-nativelink.yaml +++ b/native-cli/components/embedded/rebuild-nativelink.yaml @@ -163,3 +163,15 @@ spec: - input: "$(tasks.check-hashlocked-url.results.exists)" operator: notin values: ["true"] + + - name: update-image-tags + taskRef: + name: update-image-tags + params: + - name: imageName + value: "$(tasks.get-image-info.results.imageName)" + - name: imageTag + value: "$(tasks.get-image-info.results.imageTag)" + runAfter: + - copy-verified-prebuilt-image + - copy-nix-built-image diff --git a/native-cli/components/embedded/trigger.yaml b/native-cli/components/embedded/trigger.yaml index 1aaea3b571..229d2648db 100644 --- a/native-cli/components/embedded/trigger.yaml +++ b/native-cli/components/embedded/trigger.yaml @@ -57,7 +57,7 @@ metadata: spec: params: - name: flakeOutput - value: "$(body.flakeOutput)" + value: "$(body.metadata.flakeOutput)" --- apiVersion: v1 kind: ServiceAccount diff --git a/native-cli/components/embedded/update-image-tags.yaml b/native-cli/components/embedded/update-image-tags.yaml new file mode 100644 index 0000000000..37edd806de --- /dev/null +++ b/native-cli/components/embedded/update-image-tags.yaml @@ -0,0 +1,79 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: configmap-manager + namespace: default +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: configmap-manager-binding + namespace: default +subjects: +- kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: Role + name: configmap-manager + apiGroup: rbac.authorization.k8s.io +# --- +# apiVersion: tekton.dev/v1beta1 +# kind: Task +# metadata: +# name: update-image-tags +# labels: +# app.kubernetes.io/versions: "0.1" +# spec: +# description: > +# Create a configmap that maps +# Update image tags and kustomizations. +# params: +# - name: imageName +# - name: imageTag +# steps: +# - name: update-configmaps +# image: bitnami/kubectl +# script: | +# # Update image tags +# kubectl create configmap -n flux-system nativelink-image-tags \ +# --from-literal=$(params.imageName)=$(params.imageTag) \ +# -o yaml --dry-run=client | kubectl apply -f - +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: update-image-tags + labels: + app.kubernetes.io/versions: "0.1" +spec: + description: > + Update or add image tags in a ConfigMap. + params: + - name: imageName + - name: imageTag + steps: + - name: update-configmap + image: bitnami/kubectl + script: | + #!/bin/bash + set -e + + CM_NAME="nativelink-image-tags" + + # Transform the key to a format that flux accepts. The transformation + # here looks like `nativelink-worker` -> `NATIVELINK_WORKER_TAG`. + TRANSFORMED_KEY=$(echo "$(params.imageName)" | tr '[:lower:]' '[:upper:]' | tr '-' '_')_TAG + + # Check if the ConfigMap exists + if kubectl get configmap $CM_NAME &>/dev/null; then + # ConfigMap exists, update it + kubectl patch configmap $CM_NAME --type=json -p='[{"op": "add", "path": "/data/'$TRANSFORMED_KEY'", "value": "$(params.imageTag)"}]' + else + # ConfigMap doesn't exist, create it + kubectl create configmap $CM_NAME --from-literal=$TRANSFORMED_KEY=$(params.imageTag) + fi diff --git a/native-cli/components/flux.go b/native-cli/components/flux.go new file mode 100644 index 0000000000..0677c315a1 --- /dev/null +++ b/native-cli/components/flux.go @@ -0,0 +1,31 @@ +package components + +import ( + "fmt" + + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +// The configuration for Flux. +type Flux struct { + Version string +} + +// Install installs the Flux CRDs into the cluster.. +func (component *Flux) Install( + ctx *pulumi.Context, + name string, +) ([]pulumi.Resource, error) { + flux, err := yaml.NewConfigFile(ctx, name, &yaml.ConfigFileArgs{ + File: fmt.Sprintf( + "https://github.com/fluxcd/flux2/releases/download/v%s/install.yaml", + component.Version, + ), + }) + if err != nil { + return nil, fmt.Errorf("%w: %w", errPulumi, err) + } + + return []pulumi.Resource{flux}, nil +} diff --git a/native-cli/components/loadbalancer.go b/native-cli/components/loadbalancer.go index 7118b89a4b..61f3a421d4 100644 --- a/native-cli/components/loadbalancer.go +++ b/native-cli/components/loadbalancer.go @@ -310,6 +310,7 @@ func (component *Loadbalancer) Install( "el-gateway": false, "hubble-gateway": false, "tkn-gateway": false, + "capacitor-gateway": false, }, ), component.Gateways) if err != nil { diff --git a/native-cli/default.nix b/native-cli/default.nix index ade32b0650..c94c3db11f 100644 --- a/native-cli/default.nix +++ b/native-cli/default.nix @@ -3,7 +3,7 @@ pkgs.buildGoModule { pname = "native-cli"; version = "0.4.0"; src = ./.; - vendorHash = "sha256-zB+gaJB+5KEnkPHX2BY8nbO/oOmPk4lfmGzdPBMOSxE="; + vendorHash = "sha256-eVfeZiyyO7rUH1kDeEa41uWA0ERwCPFnOu6VTsBIEUg="; buildInputs = [pkgs.makeWrapper]; ldflags = ["-s -w"]; installPhase = '' diff --git a/native-cli/go.mod b/native-cli/go.mod index d3fa6d457c..9f048b9073 100644 --- a/native-cli/go.mod +++ b/native-cli/go.mod @@ -3,11 +3,11 @@ module github.com/TraceMachina/nativelink/native-cli go 1.22.1 require ( - github.com/docker/docker v27.0.0+incompatible + github.com/docker/docker v27.0.2+incompatible github.com/go-git/go-git/v5 v5.12.0 github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 - github.com/pulumi/pulumi/sdk/v3 v3.120.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.14.0 + github.com/pulumi/pulumi/sdk/v3 v3.121.0 github.com/spf13/cobra v1.8.1 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 @@ -29,7 +29,7 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/charmbracelet/bubbles v0.18.0 // indirect - github.com/charmbracelet/bubbletea v0.26.4 // indirect + github.com/charmbracelet/bubbletea v0.26.6 // indirect github.com/charmbracelet/lipgloss v0.11.0 // indirect github.com/charmbracelet/x/ansi v0.1.2 // indirect github.com/charmbracelet/x/input v0.1.2 // indirect @@ -68,7 +68,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.21.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -136,7 +136,7 @@ require ( golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -146,8 +146,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect k8s.io/api v0.30.2 // indirect - k8s.io/klog/v2 v2.130.0 // indirect - k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect lukechampine.com/frand v1.4.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/native-cli/go.sum b/native-cli/go.sum index b4c54ee577..8a76ee6584 100644 --- a/native-cli/go.sum +++ b/native-cli/go.sum @@ -36,8 +36,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= -github.com/charmbracelet/bubbletea v0.26.4 h1:2gDkkzLZaTjMl/dQBpNVtnvcCxsh/FCkimep7FC9c40= -github.com/charmbracelet/bubbletea v0.26.4/go.mod h1:P+r+RRA5qtI1DOHNFn0otoNwB4rn+zNAzSj/EXz6xU0= +github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= +github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= github.com/charmbracelet/lipgloss v0.11.0 h1:UoAcbQ6Qml8hDwSWs0Y1cB5TEQuZkDPH/ZqwWWYTG4g= github.com/charmbracelet/lipgloss v0.11.0/go.mod h1:1UdRTH9gYgpcdNN5oBtjbu/IzNKtzVtb7sqN1t9LNn8= github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY= @@ -65,8 +65,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/docker v27.0.0+incompatible h1:JRugTYuelmWlW0M3jakcIadDx2HUoUO6+Tf2C5jVfwA= -github.com/docker/docker v27.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.2+incompatible h1:mNhCtgXNV1fIRns102grG7rdzIsGGCq1OlOD0KunZos= +github.com/docker/docker v27.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -146,8 +146,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= +github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -244,10 +244,10 @@ github.com/pulumi/esc v0.9.1 h1:HH5eEv8sgyxSpY5a8yePyqFXzA8cvBvapfH8457+mIs= github.com/pulumi/esc v0.9.1/go.mod h1:oEJ6bOsjYlQUpjf70GiX+CXn3VBmpwFDxUTlmtUN84c= github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 h1:plWLn9O6u80Vr37LoCsckyobBfcrdTU9cERor72QjqA= github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1/go.mod h1:N4Yu4c49QErfucPt9Y/fGmpTryRqc0VfhyKHsGR9/g8= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 h1:Fp7siNqQBjwIoY/7Jaml/v1frOyGO+kYeeMrO4d2k7k= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1/go.mod h1:MZ+ci9Iq8f0K1aOTXgD3X+ENo2+dFbgQQ7Ahh0YZ8/g= -github.com/pulumi/pulumi/sdk/v3 v3.120.0 h1:KYtMkCmcSg4U+w41/Q0l3llKEodbfdyq6J0VMoEoVmY= -github.com/pulumi/pulumi/sdk/v3 v3.120.0/go.mod h1:/mQJPO+HehhoSJ9O3C6eUKAGeAr+4KSrbDhLsXHKldc= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.14.0 h1:y9+1n+Qh37zCronhamSmxa946T0ekM7VaJE1+UARIcM= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.14.0/go.mod h1:Smqya4ClBt2okzTDgPkihKu/hHtoZJNkBtUWoHUUCbw= +github.com/pulumi/pulumi/sdk/v3 v3.121.0 h1:UsnFKIVOtJN/hQKPkWHL9cZktewPVQRbNUXbXQY/qrk= +github.com/pulumi/pulumi/sdk/v3 v3.121.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -415,8 +415,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -446,10 +446,10 @@ k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= -k8s.io/klog/v2 v2.130.0 h1:5nB3+3HpqKqXJIXNtJdtxcDCfaa9KL8StJgMzGJkUkM= -k8s.io/klog/v2 v2.130.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= -k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw= +k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= diff --git a/native-cli/programs/local.go b/native-cli/programs/local.go index 09f2a22ab6..48520da147 100644 --- a/native-cli/programs/local.go +++ b/native-cli/programs/local.go @@ -83,6 +83,17 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { "tekton-dashboard", &components.TektonDashboard{Version: "0.45.0"}, )) + + flux, err := components.AddComponent( + ctx, + "flux", + &components.Flux{Version: "2.3.0"}, + ) + if err != nil { + log.Println(err) + os.Exit(1) + } + components.Check(components.AddComponent( ctx, "rebuild-nativelink", @@ -93,6 +104,7 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { tektonTriggers, localSources, nixStore, + flux, ), }, )) @@ -143,6 +155,17 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { }, } + capacitorGateway := components.Gateway{ + ExternalPort: 9000, //nolint:mnd + InternalPort: 9000, //nolint:mnd + Routes: []components.RouteConfig{ + { + Prefix: "/", + Cluster: "capacitor-gateway", + }, + }, + } + nativelinkGateway := components.Gateway{ ExternalPort: 8082, //nolint:mnd InternalPort: 8089, //nolint:mnd @@ -172,6 +195,7 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error { "kind-loadbalancer", &components.Loadbalancer{ Gateways: []components.Gateway{ + capacitorGateway, nativelinkGateway, hubbleGateway, tknGateway, diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index f0e27c752f..8e5fd5f713 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -48,7 +48,7 @@ in { ++ [ # Integration testfiles not intended for production. "deployment-examples/docker-compose/example-do-not-use-in-prod-key.pem" - "deployment-examples/kubernetes/base/example-do-not-use-in-prod-key.pem" + "kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem" ]; enable = true; name = "detect-private-key";