From 408148d8fd87f9b33ce98b11c8960afc37c18794 Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Mon, 2 Dec 2024 23:11:47 +0100 Subject: [PATCH] Implement Local Remote Execution for Rust This change implements ~40 Rust toolchain configurations that reuse Nix's executables and make them available via Bazel. The new setup is fully reproducible, supports musl and glibc and works from all nix-supported hosts to all sensible crosscompilation targets seamlessly through remote and local execution. Until we have the actual infrastructure set up to fully test all configurations we have these 12 which are practically usable immediately: - MacOS native, aarch64 and x86_64, stable and nightly - Linux native, aarch64 and x64_64, stable and nightly, glibc and musl The channel is configurable via a new `--@local-remote-execution//lre-rs:channel` flag while the libc version is configurable via the target platform. For instance: ```bash bazel build \ nativelink \ --@local-remote-execution//lre-rs:channel=nightly \ --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-gnu bazel build \ nativelink \ --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-musl ``` --- .bazelignore | 1 + .bazelrc | 5 +- .github/workflows/lre.yaml | 69 +++-- .github/workflows/main.yml | 3 + .github/workflows/native-bazel.yaml | 4 +- MODULE.bazel | 79 +++--- deploy/dev/kustomization.yaml | 42 ++- flake.nix | 114 +++----- kubernetes/base/kustomization.yaml | 12 - kubernetes/components/alerts/alert.yaml | 17 ++ .../kustomization.yaml | 3 +- .../kustomization/flux-kustomization.yaml | 22 ++ .../kustomization.yaml | 4 +- .../components/operator/flux-config.yaml | 129 --------- .../components/worker/kustomization.yaml | 18 ++ .../worker}/worker.json5 | 5 + .../{base => components/worker}/worker.yaml | 0 .../kustomization.yaml | 13 +- .../nativelink-config.json5 | 3 +- .../{base => nativelink}/nativelink.yaml | 0 .../overlays/chromium/kustomization.yaml | 3 +- .../overlays/lre-flux/kustomization.yaml | 9 + kubernetes/overlays/lre/kustomization.yaml | 45 ++-- .../alert-kustomization.yaml | 24 ++ .../alert-kustomization/kustomization.yaml | 6 + .../resources/flux/flux-kustomizations.yaml | 38 +++ kubernetes/resources/flux/gitrepository.yaml | 11 + kubernetes/resources/flux/kustomization.yaml | 7 + .../gateway-routes/kustomization.yaml | 6 + .../gateway-routes/routes.yaml | 0 .../example-do-not-use-in-prod-key.pem | 0 .../example-do-not-use-in-prod-rootca.crt | 0 .../insecure-certs/kustomization.yaml | 4 +- .../nativelink-core/kustomization.yaml | 52 ++++ .../resources/provider/kustomization.yaml | 6 + kubernetes/resources/provider/provider.yaml | 9 + .../tekton}/cosign-verify.yaml | 0 .../tekton}/kustomization.yaml | 4 +- .../tekton}/nix2container-copyto.yaml | 0 .../tekton}/nix2container-image-info.yaml | 0 .../tekton}/rebuild-nativelink.yaml | 9 + .../tekton}/skopeo-check-hashlocked-url.yaml | 0 .../tekton}/skopeo-copy.yaml | 0 .../tekton}/trigger.yaml | 10 + .../tekton}/update-image-tags.yaml | 3 +- kubernetes/workers/lre-cc/kustomization.yaml | 17 ++ .../lre => workers/lre-cc}/worker-lre-cc.yaml | 6 +- kubernetes/workers/lre-rs/kustomization.yaml | 17 ++ kubernetes/workers/lre-rs/worker-lre-rs.yaml | 48 ++++ .../lre-cc-standalone/kustomization.yaml | 7 + .../lre-rs-standalone/kustomization.yaml | 7 + .../kustomization.yaml | 8 + .../resources/lre-cc/kustomization.yaml | 60 +++++ .../resources/lre-rs/kustomization.yaml | 60 +++++ .../siso-chromium/kustomization.yaml | 60 +++++ .../resources/worker-init/kustomization.yaml | 22 ++ .../workers/siso-chromium/kustomization.yaml | 11 + local-remote-execution/MODULE.bazel | 15 ++ local-remote-execution/examples/BUILD.bazel | 1 + local-remote-execution/flake-module.nix | 9 + local-remote-execution/libc/BUILD.bazel | 13 + local-remote-execution/lre-cc.nix | 37 --- local-remote-execution/lre-rs/BUILD.bazel | 242 +++++++++++++++++ .../lre-rs/aarch64-darwin.BUILD.bazel | 125 +++++++++ .../lre-rs/aarch64-linux.BUILD.bazel | 101 +++++++ local-remote-execution/lre-rs/extension.bzl | 71 +++++ .../lre-rs/platforms/BUILD.bazel | 78 ++++++ .../lre-rs/triple/BUILD.bazel | 69 +++++ .../lre-rs/x86_64-darwin.BUILD.bazel | 125 +++++++++ .../lre-rs/x86_64-linux.BUILD.bazel | 101 +++++++ local-remote-execution/modules/lre.nix | 92 +++++-- local-remote-execution/nix-system/BUILD.bazel | 87 ++++++ local-remote-execution/overlays/default.nix | 54 +++- local-remote-execution/overlays/lre-cc.nix | 84 ++++++ local-remote-execution/overlays/lre-rs.nix | 248 ++++++++++++++++++ .../{ => overlays}/rbe-autogen.nix | 4 +- .../rbe-configs-gen/default.nix} | 0 .../rbe_configs_gen_adjustments.diff | 0 .../overlays/rust-config.nix | 76 ++++++ nativelink-config/BUILD.bazel | 4 + tools/generate-toolchains.nix | 29 -- tools/pre-commit-hooks.nix | 2 +- tools/rules_rust-musl-platforms.diff | 13 + web/platform/package.json | 2 +- .../content/docs/docs/contribute/bazel.mdx | 22 +- 85 files changed, 2360 insertions(+), 456 deletions(-) delete mode 100644 kubernetes/base/kustomization.yaml create mode 100644 kubernetes/components/alerts/alert.yaml rename kubernetes/components/{gateway-routes => alerts}/kustomization.yaml (82%) create mode 100644 kubernetes/components/kustomization/flux-kustomization.yaml rename kubernetes/components/{operator => kustomization}/kustomization.yaml (68%) delete mode 100644 kubernetes/components/operator/flux-config.yaml create mode 100644 kubernetes/components/worker/kustomization.yaml rename kubernetes/{configmaps => components/worker}/worker.json5 (95%) rename kubernetes/{base => components/worker}/worker.yaml (100%) rename kubernetes/{configmaps => nativelink}/kustomization.yaml (64%) rename kubernetes/{configmaps => nativelink}/nativelink-config.json5 (98%) rename kubernetes/{base => nativelink}/nativelink.yaml (100%) create mode 100644 kubernetes/overlays/lre-flux/kustomization.yaml create mode 100644 kubernetes/resources/alert-kustomization/alert-kustomization.yaml create mode 100644 kubernetes/resources/alert-kustomization/kustomization.yaml create mode 100644 kubernetes/resources/flux/flux-kustomizations.yaml create mode 100644 kubernetes/resources/flux/gitrepository.yaml create mode 100644 kubernetes/resources/flux/kustomization.yaml create mode 100644 kubernetes/resources/gateway-routes/kustomization.yaml rename kubernetes/{components => resources}/gateway-routes/routes.yaml (100%) rename kubernetes/{components => resources}/insecure-certs/example-do-not-use-in-prod-key.pem (100%) rename kubernetes/{components => resources}/insecure-certs/example-do-not-use-in-prod-rootca.crt (100%) rename kubernetes/{components => resources}/insecure-certs/kustomization.yaml (69%) create mode 100644 kubernetes/resources/nativelink-core/kustomization.yaml create mode 100644 kubernetes/resources/provider/kustomization.yaml create mode 100644 kubernetes/resources/provider/provider.yaml rename kubernetes/{components/tekton-resources => resources/tekton}/cosign-verify.yaml (100%) rename kubernetes/{components/tekton-resources => resources/tekton}/kustomization.yaml (78%) rename kubernetes/{components/tekton-resources => resources/tekton}/nix2container-copyto.yaml (100%) rename kubernetes/{components/tekton-resources => resources/tekton}/nix2container-image-info.yaml (100%) rename kubernetes/{components/tekton-resources => resources/tekton}/rebuild-nativelink.yaml (94%) rename kubernetes/{components/tekton-resources => resources/tekton}/skopeo-check-hashlocked-url.yaml (100%) rename kubernetes/{components/tekton-resources => resources/tekton}/skopeo-copy.yaml (100%) rename kubernetes/{components/tekton-resources => resources/tekton}/trigger.yaml (90%) rename kubernetes/{components/tekton-resources => resources/tekton}/update-image-tags.yaml (94%) create mode 100644 kubernetes/workers/lre-cc/kustomization.yaml rename kubernetes/{overlays/lre => workers/lre-cc}/worker-lre-cc.yaml (88%) create mode 100644 kubernetes/workers/lre-rs/kustomization.yaml create mode 100644 kubernetes/workers/lre-rs/worker-lre-rs.yaml create mode 100644 kubernetes/workers/overlays/lre-cc-standalone/kustomization.yaml create mode 100644 kubernetes/workers/overlays/lre-rs-standalone/kustomization.yaml create mode 100644 kubernetes/workers/overlays/siso-chromium-standalone/kustomization.yaml create mode 100644 kubernetes/workers/resources/lre-cc/kustomization.yaml create mode 100644 kubernetes/workers/resources/lre-rs/kustomization.yaml create mode 100644 kubernetes/workers/resources/siso-chromium/kustomization.yaml create mode 100644 kubernetes/workers/resources/worker-init/kustomization.yaml create mode 100644 kubernetes/workers/siso-chromium/kustomization.yaml create mode 100644 local-remote-execution/libc/BUILD.bazel delete mode 100644 local-remote-execution/lre-cc.nix create mode 100644 local-remote-execution/lre-rs/BUILD.bazel create mode 100644 local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel create mode 100644 local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel create mode 100644 local-remote-execution/lre-rs/extension.bzl create mode 100644 local-remote-execution/lre-rs/platforms/BUILD.bazel create mode 100644 local-remote-execution/lre-rs/triple/BUILD.bazel create mode 100644 local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel create mode 100644 local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel create mode 100644 local-remote-execution/nix-system/BUILD.bazel create mode 100644 local-remote-execution/overlays/lre-cc.nix create mode 100644 local-remote-execution/overlays/lre-rs.nix rename local-remote-execution/{ => overlays}/rbe-autogen.nix (98%) rename local-remote-execution/{rbe-configs-gen.nix => overlays/rbe-configs-gen/default.nix} (100%) rename local-remote-execution/{ => overlays/rbe-configs-gen}/rbe_configs_gen_adjustments.diff (100%) create mode 100644 local-remote-execution/overlays/rust-config.nix create mode 100644 tools/rules_rust-musl-platforms.diff diff --git a/.bazelignore b/.bazelignore index c52920b24..bf4b8b95f 100644 --- a/.bazelignore +++ b/.bazelignore @@ -13,3 +13,4 @@ bazel-testlogs bazel-nativelink local-remote-execution/generated-cc local-remote-execution/generated-java +local-remote-execution/lre-rs diff --git a/.bazelrc b/.bazelrc index fd9a1d20f..a1b212f59 100644 --- a/.bazelrc +++ b/.bazelrc @@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh startup --windows_enable_symlinks build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14 build:windows --enable_runfiles - -# Global rust toolchain configuration. Deferred to here so that the cc -# toolchains are resolved before the rust toolchains. -build --extra_toolchains=@rust_toolchains//:all +build:windows --extra_toolchains=@rust_toolchains//:all # Doesn't support LRE. # Generated by the LRE flake module. try-import %workspace%/lre.bazelrc diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index 478b848f3..fe9a827e9 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -96,16 +96,9 @@ jobs: nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - components: - - kubernetes/components/operator + resources: + - kubernetes/overlays/lre-flux patches: - - patch: |- - - op: replace - path: /spec/path - value: ./kubernetes/overlays/lre - target: - kind: Kustomization - name: nativelink - patch: |- - op: replace path: /spec/url @@ -119,27 +112,6 @@ jobs: target: kind: GitRepository name: nativelink - - patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: ./src_root#image - target: - kind: Alert - name: nativelink-image-alert - - patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: ./src_root#nativelink-worker-init - target: - kind: Alert - name: nativelink-worker-init-alert - - patch: |- - - op: replace - path: /spec/eventMetadata/flakeOutput - value: ./src_root#nativelink-worker-lre-cc - target: - kind: Alert - name: nativelink-worker-alert EOF kubectl apply -k . && rm kustomization.yaml' @@ -151,6 +123,36 @@ jobs: --timeout=15m \ nativelink-tekton-resources" + - name: Wait for alerts + run: > + nix develop --impure --command + bash -c "flux reconcile kustomization -n default \ + --timeout=15m \ + nativelink-alert-core && \ + flux reconcile kustomization -n default \ + --timeout=15m \ + nativelink-alert-worker-init && \ + flux reconcile kustomization -n default \ + --timeout=15m \ + nativelink-alert-lre-cc" + + - name: Trigger pipelines + run: | + nix develop --impure --command bash -c 'cat > dummy-repo.yaml << EOF + apiVersion: source.toolkit.fluxcd.io/v1 + kind: GitRepository + metadata: + name: dummy-repository + namespace: default + spec: + interval: 2m + url: https://github.com/TraceMachina/nativelink + ref: + branch: main + EOF + kubectl apply -f dummy-repo.yaml && + rm dummy-repo.yaml' + - name: Wait for Tekton pipelines run: > nix develop --impure --command bash << 'EOF' @@ -171,13 +173,6 @@ jobs: -l tekton.dev/pipeline=rebuild-nativelink EOF - - name: Wait for Configmaps - run: > - nix develop --impure --command - bash -c "flux reconcile kustomization -n default \ - --timeout=15m \ - nativelink-configmaps" - - name: Wait for NativeLink Kustomization run: > nix develop --impure --command diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff698c82c..0a467c39c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -159,6 +159,7 @@ jobs: docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \ bazel clean && \ bazel test //... \ + --extra_toolchains=@rust_toolchains//:all \ --remote_cache=grpc://127.0.0.1:50051 \ --remote_executor=grpc://127.0.0.1:50052 \ --remote_default_exec_properties=cpu_count=1 \ @@ -166,6 +167,7 @@ jobs: docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \ bazel clean && \ bazel test //... \ + --extra_toolchains=@rust_toolchains//:all \ --remote_cache=grpc://127.0.0.1:50051 \ --remote_executor=grpc://127.0.0.1:50052 \ --remote_default_exec_properties=cpu_count=1 \ @@ -195,6 +197,7 @@ jobs: build-args: | OPT_LEVEL=fastbuild OS_VERSION=${{ matrix.os_version }} + ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all load: true # This brings the build into `docker images` from buildx. tags: trace_machina/nativelink:latest diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml index b01c09898..8c588e08e 100644 --- a/.github/workflows/native-bazel.yaml +++ b/.github/workflows/native-bazel.yaml @@ -72,7 +72,9 @@ jobs: - name: Run Bazel tests run: | if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then - bazel test //... --verbose_failures + bazel test //... \ + --extra_toolchains=@rust_toolchains//:all \ + --verbose_failures elif [ "$RUNNER_OS" == "Windows" ]; then bazel \ --output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \ diff --git a/MODULE.bazel b/MODULE.bazel index b40d261fb..ecc336b15 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( bazel_dep(name = "rules_cc", version = "0.0.17") bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "rules_python", version = "0.36.0") +bazel_dep(name = "rules_python", version = "0.40.0") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( @@ -19,10 +19,50 @@ python.toolchain( use_repo(python, python = "python_versions") bazel_dep(name = "rules_rust", version = "0.54.1") +archive_override( + module_name = "rules_rust", + integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=", + patch_strip = 1, + patches = ["//tools:rules_rust-musl-platforms.diff"], + urls = [ + "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz", + ], +) + +crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate") +crate.from_cargo( + name = "crates", + cargo_lockfile = "//:Cargo.lock", + manifests = ["//:Cargo.toml"], + supported_platform_triples = [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "armv7-unknown-linux-gnueabi", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + ], +) +use_repo(crate, "crates") +# NativeLink uses Local Remote Execution for Rust by default which automatically +# handles Rust toolchain configuration via Nix. +# +# If you build outside of Nix you'll have to register these toolchains +# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your +# Bazel invocation. +# +# WARNING: This configuration exists entirely as a convenience option and +# migration and is not a supported way of building production +# grade nativelink executables. It may be removed at any point in time. rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") rust.toolchain( edition = "2021", + + # These should always follow the versions from LRE. rust_analyzer_version = "nightly/2024-11-23", rustfmt_version = "nightly/2024-11-23", sha256s = { @@ -44,43 +84,10 @@ rust.toolchain( "nightly/2024-11-23", ], ) - -rust_host_tools = use_extension( - "@rules_rust//rust:extension.bzl", - "rust_host_tools", -) -rust_host_tools.host_tools( - edition = "2021", - version = "1.82.0", -) - use_repo(rust, "rust_toolchains") -crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate") -crate.from_cargo( - name = "crates", - cargo_lockfile = "//:Cargo.lock", - manifests = ["//:Cargo.toml"], - supported_platform_triples = [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "arm-unknown-linux-gnueabi", - "armv7-unknown-linux-gnueabi", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", - ], -) -use_repo(crate, "crates") - -rust_analyzer = use_extension( - "@rules_rust//tools/rust_analyzer:extension.bzl", - "rust_analyzer_dependencies", -) -rust_analyzer.rust_analyzer_dependencies() - -bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf") -bazel_dep(name = "toolchains_protoc", version = "0.3.3") +bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf") +bazel_dep(name = "toolchains_protoc", version = "0.3.4") protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc") protoc.toolchain( diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml index 35a3f183a..bc233c928 100644 --- a/deploy/dev/kustomization.yaml +++ b/deploy/dev/kustomization.yaml @@ -1,33 +1,23 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -components: -- ../../kubernetes/components/operator +resources: +- ../../kubernetes/overlays/lre-flux -# Change this value to deploy custom overlays. -patches: -- patch: |- - - op: replace - path: /spec/path - value: ./kubernetes/overlays/lre - target: - kind: Kustomization - name: nativelink - -# Modify this value to change the URL of the repository with deployment files. +# Modify this value to change the repository containing the deployment files. # # This is usually only necessary if you change deployment YAML files or # NativeLink config files. If you only intend to change the Rust sources you can # leave this as is and need to ensure that the Alerts below are patched to build # your local sources. +patches: - patch: |- - op: replace path: /spec/url value: https://github.com/TraceMachina/nativelink -# Optionally, change the tracked branch. -# - op: replace -# path: /spec/ref/branch -# value: somecustombranch + - op: replace + path: /spec/ref/branch + value: main target: kind: GitRepository name: nativelink @@ -43,22 +33,22 @@ patches: # outputs from a Pull request. - patch: |- - op: replace - path: /spec/eventMetadata/flakeOutput + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT value: ./src_root#image target: - kind: Alert - name: nativelink-image-alert + kind: Kustomization + name: nativelink-alert-core - patch: |- - op: replace - path: /spec/eventMetadata/flakeOutput + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT value: ./src_root#nativelink-worker-init target: - kind: Alert - name: nativelink-worker-init-alert + kind: Kustomization + name: nativelink-alert-worker-init - patch: |- - op: replace - path: /spec/eventMetadata/flakeOutput + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT value: ./src_root#nativelink-worker-lre-cc target: - kind: Alert - name: nativelink-worker-alert + kind: Kustomization + name: nativelink-alert-lre-cc diff --git a/flake.nix b/flake.nix index 2738191ff..9b4b6dba1 100644 --- a/flake.nix +++ b/flake.nix @@ -52,54 +52,8 @@ system, ... }: let - stable-rust-version = "1.82.0"; - nightly-rust-version = "2024-11-23"; - - llvmPackages = pkgs.llvmPackages_19; - - nixSystemToRustTriple = nixSystem: - { - "x86_64-linux" = "x86_64-unknown-linux-musl"; - "aarch64-linux" = "aarch64-unknown-linux-musl"; - "x86_64-darwin" = "x86_64-apple-darwin"; - "aarch64-darwin" = "aarch64-apple-darwin"; - } - .${nixSystem} - or (throw "Unsupported Nix system: ${nixSystem}"); - - # Calling `pkgs.pkgsCross` changes the host and target platform to the - # cross-target but leaves the build platform the same as pkgs. - # - # For instance, calling `pkgs.pkgsCross.aarch64-multiplatform` on an - # `x86_64-linux` host sets `host==target==aarch64-linux` but leaves the - # build platform at `x86_64-linux`. - # - # On aarch64-darwin the same `pkgs.pkgsCross.aarch64-multiplatform` - # again sets `host==target==aarch64-linux` but now with a build platform - # of `aarch64-darwin`. - # - # For optimal cache reuse of different crosscompilation toolchains we - # take our rust toolchain from the host's `pkgs` and remap the rust - # target to the target platform of the `pkgsCross` target. This lets us - # reuse the same executables (for instance rustc) to build artifacts for - # different target platforms. - stableRustFor = p: - p.rust-bin.stable.${stable-rust-version}.default.override { - targets = [ - "${nixSystemToRustTriple p.stdenv.targetPlatform.system}" - ]; - }; - - nightlyRustFor = p: - p.rust-bin.nightly.${nightly-rust-version}.default.override { - extensions = ["llvm-tools"]; - targets = [ - "${nixSystemToRustTriple p.stdenv.targetPlatform.system}" - ]; - }; - - craneLibFor = p: (crane.mkLib p).overrideToolchain stableRustFor; - nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain nightlyRustFor; + craneLibFor = p: (crane.mkLib p).overrideToolchain pkgs.lre.stableRustFor; + nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain pkgs.lre.nightlyRustFor; src = pkgs.lib.cleanSourceWith { src = (craneLibFor pkgs).path ./.; @@ -110,18 +64,32 @@ # Warning: The different usages of `p` and `pkgs` are intentional as we # use crosscompilers and crosslinkers whose packagesets collapse with - # the host's packageset. If you change this, take care that you don't + # the host's packageset. If you change this take care that you don't # accidentally explode the global closure size. commonArgsFor = p: let isLinuxBuild = p.stdenv.buildPlatform.isLinux; isLinuxTarget = p.stdenv.targetPlatform.isLinux; - targetArch = nixSystemToRustTriple p.stdenv.targetPlatform.system; + # Map the nix system to the Rust target triple that we'd want to target + # by default. + targetArch = + ( + nixSystem: + { + "x86_64-linux" = "x86_64-unknown-linux-musl"; + "aarch64-linux" = "aarch64-unknown-linux-musl"; + "x86_64-darwin" = "x86_64-apple-darwin"; + "aarch64-darwin" = "aarch64-apple-darwin"; + } + .${nixSystem} + or (throw "Unsupported Nix host platform: ${nixSystem}") + ) + p.stdenv.targetPlatform.system; # Full path to the linker for CARGO_TARGET_XXX_LINKER linkerPath = if isLinuxBuild && isLinuxTarget then "${pkgs.mold}/bin/ld.mold" - else "${llvmPackages.lld}/bin/ld.lld"; + else "${pkgs.llvmPackages_19.lld}/bin/ld.lld"; linkerEnvVar = "CARGO_TARGET_${pkgs.lib.toUpper (pkgs.lib.replaceStrings ["-"] ["_"] targetArch)}_LINKER"; in @@ -142,7 +110,7 @@ ( if isLinuxBuild then [pkgs.mold] - else [llvmPackages.lld] + else [pkgs.llvmPackages_19.lld] ) ++ pkgs.lib.optionals p.stdenv.targetPlatform.isDarwin [ p.darwin.apple_sdk.frameworks.Security @@ -193,10 +161,6 @@ nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;}; - rbe-configs-gen = pkgs.callPackage ./local-remote-execution/rbe-configs-gen.nix {}; - - generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {inherit rbe-configs-gen;}; - native-cli = pkgs.callPackage ./native-cli/default.nix {}; build-chromium-tests = @@ -204,7 +168,7 @@ "build-chromium-tests" ./deploy/chromium-example/build_chromium_tests.sh; - docs = pkgs.callPackage ./tools/docs.nix {rust = stableRustFor pkgs;}; + docs = pkgs.callPackage ./tools/docs.nix {rust = pkgs.lre.stable-rust;}; inherit (nix2container.packages.${system}.nix2container) pullImage; inherit (nix2container.packages.${system}.nix2container) buildImage; @@ -243,10 +207,6 @@ nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;}; - rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix { - inherit buildImage; - inherit (pkgs.lre) stdenv; - }; createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;}; buck2-toolchain = let buck2-nightly-rust-version = "2024-04-28"; @@ -285,9 +245,7 @@ os = "linux"; }; }; - lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix { - inherit buildImage; - }; + toolchain-drake = buildImage { name = "toolchain-drake"; # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds. @@ -364,7 +322,6 @@ rec { inherit local-image-test - lre-cc native-cli nativelink nativelinkCoverageForHost @@ -376,13 +333,14 @@ nativelink-x86_64-linux publish-ghcr ; + default = nativelink; - rbe-autogen-lre-cc = rbe-autogen lre-cc; - nativelink-worker-lre-cc = createWorker lre-cc; + nativelink-worker-lre-cc = createWorker pkgs.lre.lre-cc.image; lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;}; - rbe-autogen-lre-java = rbe-autogen lre-java; + rbe-autogen-lre-java = pkgs.rbe-autogen lre-java; nativelink-worker-lre-java = createWorker lre-java; + nativelink-worker-lre-rs = createWorker pkgs.lre.lre-rs.image; nativelink-worker-siso-chromium = createWorker siso-chromium; nativelink-worker-toolchain-drake = createWorker toolchain-drake; nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2; @@ -416,15 +374,18 @@ pre-commit.settings = { hooks = import ./tools/pre-commit-hooks.nix { inherit pkgs; - nightly-rust = pkgs.rust-bin.nightly.${nightly-rust-version}; + nightly-rust = pkgs.rust-bin.nightly.${pkgs.lre.nightly-rust.meta.version}; }; }; local-remote-execution.settings = { - Env = + Env = with pkgs.lre; + if pkgs.stdenv.isDarwin + then lre-rs.meta.Env # C++ doesn't support Darwin yet. + else (lre-cc.meta.Env ++ lre-rs.meta.Env); + prefix = if pkgs.stdenv.isDarwin - then [] # Doesn't support Darwin yet. - else lre-cc.meta.Env; - prefix = "linux"; + then "macos" + else "linux"; }; nixos.settings = { path = with pkgs; [ @@ -449,8 +410,9 @@ pkgs.pre-commit # Rust - (stableRustFor pkgs) bazel + pkgs.lre.stable-rust + pkgs.lre.lre-rs.lre-rs-configs-gen ## Infrastructure pkgs.awscli2 @@ -481,7 +443,7 @@ # Additional tools from within our development environment. local-image-test - generate-toolchains + pkgs.lre.lre-cc.lre-cc-configs-gen pkgs.lre.clang native-cli docs @@ -546,7 +508,7 @@ nixos = ./tools/nixos/flake-module.nix; }; overlays = { - lre = import ./local-remote-execution/overlays/default.nix; + lre = import ./local-remote-execution/overlays/default.nix {inherit nix2container;}; }; }; } diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml deleted file mode 100644 index 3a160e2ac..000000000 --- a/kubernetes/base/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - nativelink.yaml - - worker.yaml - - ../configmaps - -images: - - name: nativelink - - name: nativelink-worker-init - - name: nativelink-worker diff --git a/kubernetes/components/alerts/alert.yaml b/kubernetes/components/alerts/alert.yaml new file mode 100644 index 000000000..3450aa70b --- /dev/null +++ b/kubernetes/components/alerts/alert.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Alert +metadata: + name: nativelink-alert + namespace: default +spec: + eventSeverity: info + eventSources: + - kind: GitRepository + name: '*' + namespace: default + providerRef: + name: nativelink-webhook + eventMetadata: + flakeOutput: ${PLACEHOLDER_FLAKE_OUTPUT} + configMapName: ${PLACEHOLDER_CONFIG_MAP_NAME} diff --git a/kubernetes/components/gateway-routes/kustomization.yaml b/kubernetes/components/alerts/kustomization.yaml similarity index 82% rename from kubernetes/components/gateway-routes/kustomization.yaml rename to kubernetes/components/alerts/kustomization.yaml index 42b112b7e..1faa9eaf5 100644 --- a/kubernetes/components/gateway-routes/kustomization.yaml +++ b/kubernetes/components/alerts/kustomization.yaml @@ -1,5 +1,6 @@ --- apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component + resources: - - routes.yaml +- alert.yaml diff --git a/kubernetes/components/kustomization/flux-kustomization.yaml b/kubernetes/components/kustomization/flux-kustomization.yaml new file mode 100644 index 000000000..15d3f2034 --- /dev/null +++ b/kubernetes/components/kustomization/flux-kustomization.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + path: PLACEHOLDER_NATIVELINK_KUSTOMIZATION + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + postBuild: + substituteFrom: + - kind: ConfigMap + name: nativelink-image-tags diff --git a/kubernetes/components/operator/kustomization.yaml b/kubernetes/components/kustomization/kustomization.yaml similarity index 68% rename from kubernetes/components/operator/kustomization.yaml rename to kubernetes/components/kustomization/kustomization.yaml index 916d215e1..91474583a 100644 --- a/kubernetes/components/operator/kustomization.yaml +++ b/kubernetes/components/kustomization/kustomization.yaml @@ -1,4 +1,6 @@ +--- apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component + resources: -- flux-config.yaml + - flux-kustomization.yaml diff --git a/kubernetes/components/operator/flux-config.yaml b/kubernetes/components/operator/flux-config.yaml deleted file mode 100644 index cd6305a9a..000000000 --- a/kubernetes/components/operator/flux-config.yaml +++ /dev/null @@ -1,129 +0,0 @@ ---- -apiVersion: source.toolkit.fluxcd.io/v1 -kind: GitRepository -metadata: - name: nativelink - namespace: default -spec: - interval: 2m - url: https://github.com/TraceMachina/nativelink - ref: - branch: main ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Provider -metadata: - name: nativelink-webhook - namespace: flux-system -spec: - type: generic - address: http://el-nativelink-rebuild.default.svc.cluster.local:8080 ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Alert -metadata: - name: nativelink-image-alert - namespace: flux-system -spec: - eventSeverity: info - eventSources: - - kind: GitRepository - name: '*' - namespace: default - providerRef: - name: nativelink-webhook - eventMetadata: - flakeOutput: github:TraceMachina/nativelink#image ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Alert -metadata: - name: nativelink-worker-init-alert - namespace: flux-system -spec: - eventSeverity: info - eventSources: - - kind: GitRepository - name: '*' - namespace: default - providerRef: - name: nativelink-webhook - eventMetadata: - flakeOutput: github:TraceMachina/nativelink#nativelink-worker-init ---- -apiVersion: notification.toolkit.fluxcd.io/v1beta3 -kind: Alert -metadata: - name: nativelink-worker-alert - namespace: flux-system -spec: - eventSeverity: info - eventSources: - - kind: GitRepository - name: '*' - namespace: default - providerRef: - name: nativelink-webhook - eventMetadata: - flakeOutput: "PLACEHOLDER_NATIVELINK_WORKER" ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: nativelink-configmaps - namespace: default -spec: - interval: 2m - path: "./kubernetes/configmaps" - prune: true - force: true - retryInterval: 20s - targetNamespace: default - wait: true - sourceRef: - kind: GitRepository - name: nativelink - namespace: default ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: nativelink - namespace: default -spec: - interval: 2m - path: "PLACEHOLDER_NATIVELINK_KUSTOMIZATION" - prune: true - force: true - retryInterval: 20s - targetNamespace: default - wait: true - sourceRef: - kind: GitRepository - name: nativelink - namespace: default - postBuild: - substituteFrom: - - kind: ConfigMap - name: nativelink-image-tags - dependsOn: - - name: nativelink-configmaps - - name: nativelink-tekton-resources ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: nativelink-tekton-resources - namespace: default -spec: - interval: 2m - path: "./kubernetes/components/tekton-resources" - prune: true - force: true - retryInterval: 20s - targetNamespace: default - wait: true - sourceRef: - kind: GitRepository - name: nativelink - namespace: default diff --git a/kubernetes/components/worker/kustomization.yaml b/kubernetes/components/worker/kustomization.yaml new file mode 100644 index 000000000..fccbe7f19 --- /dev/null +++ b/kubernetes/components/worker/kustomization.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- worker.yaml + +configMapGenerator: + - name: worker + files: + - worker.json5 + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + +images: + - name: nativelink-worker-init + - name: nativelink-worker diff --git a/kubernetes/configmaps/worker.json5 b/kubernetes/components/worker/worker.json5 similarity index 95% rename from kubernetes/configmaps/worker.json5 rename to kubernetes/components/worker/worker.json5 index 21dcb6f95..284a6beef 100644 --- a/kubernetes/configmaps/worker.json5 +++ b/kubernetes/components/worker/worker.json5 @@ -73,6 +73,11 @@ // `local-remote-execution/generated-cc/config/BUILD`. "${NATIVELINK_WORKER_PLATFORM:-undefined_platform}" ] + }, + "lre-rs": { + "values": [ + "${NATIVELINK_WORKER_LRE_RS:-undefined}" + ] } } } diff --git a/kubernetes/base/worker.yaml b/kubernetes/components/worker/worker.yaml similarity index 100% rename from kubernetes/base/worker.yaml rename to kubernetes/components/worker/worker.yaml diff --git a/kubernetes/configmaps/kustomization.yaml b/kubernetes/nativelink/kustomization.yaml similarity index 64% rename from kubernetes/configmaps/kustomization.yaml rename to kubernetes/nativelink/kustomization.yaml index 98d8c589d..db942f434 100644 --- a/kubernetes/configmaps/kustomization.yaml +++ b/kubernetes/nativelink/kustomization.yaml @@ -2,6 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +resources: + - nativelink.yaml + - ../resources/insecure-certs + configMapGenerator: - name: nativelink-config files: @@ -9,9 +13,6 @@ configMapGenerator: options: annotations: kustomize.toolkit.fluxcd.io/substitute: disabled - - name: worker - files: - - worker.json5 - options: - annotations: - kustomize.toolkit.fluxcd.io/substitute: disabled + +images: + - name: nativelink diff --git a/kubernetes/configmaps/nativelink-config.json5 b/kubernetes/nativelink/nativelink-config.json5 similarity index 98% rename from kubernetes/configmaps/nativelink-config.json5 rename to kubernetes/nativelink/nativelink-config.json5 index 528b625e6..e06d78269 100644 --- a/kubernetes/configmaps/nativelink-config.json5 +++ b/kubernetes/nativelink/nativelink-config.json5 @@ -63,7 +63,8 @@ "cpu_model": "priority", "kernel_version": "priority", "OSFamily": "priority", - "container-image": "priority" + "container-image": "priority", + "lre-rs": "priority" } } } diff --git a/kubernetes/base/nativelink.yaml b/kubernetes/nativelink/nativelink.yaml similarity index 100% rename from kubernetes/base/nativelink.yaml rename to kubernetes/nativelink/nativelink.yaml diff --git a/kubernetes/overlays/chromium/kustomization.yaml b/kubernetes/overlays/chromium/kustomization.yaml index 10837536b..3130e113c 100644 --- a/kubernetes/overlays/chromium/kustomization.yaml +++ b/kubernetes/overlays/chromium/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../base +- ../../workers/siso-chromium components: - ../../components/gateway-routes @@ -15,6 +16,6 @@ images: - name: nativelink-worker-init newName: localhost:5001/nativelink-worker-init newTag: ${NATIVELINK_WORKER_INIT_TAG} - - name: nativelink-worker + - name: nativelink-worker-siso-chromium newName: localhost:5001/nativelink-worker-siso-chromium newTag: ${NATIVELINK_WORKER_SISO_CHROMIUM_TAG} diff --git a/kubernetes/overlays/lre-flux/kustomization.yaml b/kubernetes/overlays/lre-flux/kustomization.yaml new file mode 100644 index 000000000..efbed6431 --- /dev/null +++ b/kubernetes/overlays/lre-flux/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../resources/flux +- ../../resources/nativelink-core +- ../../resources/gateway-routes +- ../../workers/resources/worker-init +- ../../workers/resources/lre-cc diff --git a/kubernetes/overlays/lre/kustomization.yaml b/kubernetes/overlays/lre/kustomization.yaml index 707804b4e..0d8351862 100644 --- a/kubernetes/overlays/lre/kustomization.yaml +++ b/kubernetes/overlays/lre/kustomization.yaml @@ -1,26 +1,31 @@ --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: -- ../../base -components: -- ../../components/gateway-routes -- ../../components/insecure-certs +resources: + - ../../nativelink + - ../../workers/resources/lre-cc -patches: -- path: worker-lre-cc.yaml - target: - kind: Deployment - name: nativelink-worker -images: - - name: nativelink - newName: localhost:5001/nativelink - newTag: ${NATIVELINK_TAG} - - name: nativelink-worker-init - newName: localhost:5001/nativelink-worker-init - newTag: ${NATIVELINK_WORKER_INIT_TAG} - - name: nativelink-worker - newName: localhost:5001/nativelink-worker-lre-cc - newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} +# resources: +# - ../../base +# - ../../workers/lre-cc +# - ../../workers/lre-rs +# +# components: +# - ../../components/gateway-routes +# - ../../components/insecure-certs +# +# images: +# - name: nativelink +# newName: localhost:5001/nativelink +# newTag: ${NATIVELINK_TAG} +# - name: nativelink-worker-init +# newName: localhost:5001/nativelink-worker-init +# newTag: ${NATIVELINK_WORKER_INIT_TAG} +# - name: nativelink-worker-lre-cc +# newName: localhost:5001/nativelink-worker-lre-cc +# newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} +# - name: nativelink-worker-lre-rs +# newName: localhost:5001/nativelink-worker-lre-rs +# newTag: ${NATIVELINK_WORKER_LRE_RS_TAG} diff --git a/kubernetes/resources/alert-kustomization/alert-kustomization.yaml b/kubernetes/resources/alert-kustomization/alert-kustomization.yaml new file mode 100644 index 000000000..470912758 --- /dev/null +++ b/kubernetes/resources/alert-kustomization/alert-kustomization.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink-alert + namespace: default +spec: + interval: 2m + path: ./kubernetes/components/alerts + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + postBuild: + substitute: + PLACEHOLDER_FLAKE_OUTPUT: SET_BY_FLUX + PLACEHOLDER_CONFIG_MAP_NAME: SET_BY_FLUX + dependsOn: + - name: provider diff --git a/kubernetes/resources/alert-kustomization/kustomization.yaml b/kubernetes/resources/alert-kustomization/kustomization.yaml new file mode 100644 index 000000000..e6922e332 --- /dev/null +++ b/kubernetes/resources/alert-kustomization/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - alert-kustomization.yaml diff --git a/kubernetes/resources/flux/flux-kustomizations.yaml b/kubernetes/resources/flux/flux-kustomizations.yaml new file mode 100644 index 000000000..9f23a41b5 --- /dev/null +++ b/kubernetes/resources/flux/flux-kustomizations.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: provider + namespace: default +spec: + interval: 2m + path: "./kubernetes/resources/provider" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default + dependsOn: + - name: nativelink-tekton-resources +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: nativelink-tekton-resources + namespace: default +spec: + interval: 2m + path: "./kubernetes/resources/tekton" + prune: true + force: true + retryInterval: 20s + targetNamespace: default + wait: true + sourceRef: + kind: GitRepository + name: nativelink + namespace: default diff --git a/kubernetes/resources/flux/gitrepository.yaml b/kubernetes/resources/flux/gitrepository.yaml new file mode 100644 index 000000000..caf090fc3 --- /dev/null +++ b/kubernetes/resources/flux/gitrepository.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: nativelink + namespace: default +spec: + interval: 2m + url: https://github.com/aaronmondal/nativelink + ref: + branch: lre-rs diff --git a/kubernetes/resources/flux/kustomization.yaml b/kubernetes/resources/flux/kustomization.yaml new file mode 100644 index 000000000..474172b82 --- /dev/null +++ b/kubernetes/resources/flux/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - gitrepository.yaml + - flux-kustomizations.yaml diff --git a/kubernetes/resources/gateway-routes/kustomization.yaml b/kubernetes/resources/gateway-routes/kustomization.yaml new file mode 100644 index 000000000..4e8451152 --- /dev/null +++ b/kubernetes/resources/gateway-routes/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - routes.yaml diff --git a/kubernetes/components/gateway-routes/routes.yaml b/kubernetes/resources/gateway-routes/routes.yaml similarity index 100% rename from kubernetes/components/gateway-routes/routes.yaml rename to kubernetes/resources/gateway-routes/routes.yaml diff --git a/kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem b/kubernetes/resources/insecure-certs/example-do-not-use-in-prod-key.pem similarity index 100% rename from kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem rename to kubernetes/resources/insecure-certs/example-do-not-use-in-prod-key.pem diff --git a/kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt b/kubernetes/resources/insecure-certs/example-do-not-use-in-prod-rootca.crt similarity index 100% rename from kubernetes/components/insecure-certs/example-do-not-use-in-prod-rootca.crt rename to kubernetes/resources/insecure-certs/example-do-not-use-in-prod-rootca.crt diff --git a/kubernetes/components/insecure-certs/kustomization.yaml b/kubernetes/resources/insecure-certs/kustomization.yaml similarity index 69% rename from kubernetes/components/insecure-certs/kustomization.yaml rename to kubernetes/resources/insecure-certs/kustomization.yaml index bc92cb026..203e442c1 100644 --- a/kubernetes/components/insecure-certs/kustomization.yaml +++ b/kubernetes/resources/insecure-certs/kustomization.yaml @@ -1,6 +1,6 @@ --- -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization secretGenerator: - name: tls-secret diff --git a/kubernetes/resources/nativelink-core/kustomization.yaml b/kubernetes/resources/nativelink-core/kustomization.yaml new file mode 100644 index 000000000..a4aa1b2ea --- /dev/null +++ b/kubernetes/resources/nativelink-core/kustomization.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -core + +resources: + - ../../resources/alert-kustomization + +components: + - ../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#image + - op: add + path: /spec/nameSuffix + value: -core + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/nativelink + - op: add + path: /spec/images + value: + - name: nativelink + newName: localhost:5001/nativelink + newTag: ${NATIVELINK_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/resources/provider/kustomization.yaml b/kubernetes/resources/provider/kustomization.yaml new file mode 100644 index 000000000..353ca7f5d --- /dev/null +++ b/kubernetes/resources/provider/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - provider.yaml diff --git a/kubernetes/resources/provider/provider.yaml b/kubernetes/resources/provider/provider.yaml new file mode 100644 index 000000000..b99e8f4af --- /dev/null +++ b/kubernetes/resources/provider/provider.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: nativelink-webhook + namespace: default +spec: + type: generic + address: http://el-nativelink-rebuild.default.svc.cluster.local:8080 diff --git a/kubernetes/components/tekton-resources/cosign-verify.yaml b/kubernetes/resources/tekton/cosign-verify.yaml similarity index 100% rename from kubernetes/components/tekton-resources/cosign-verify.yaml rename to kubernetes/resources/tekton/cosign-verify.yaml diff --git a/kubernetes/components/tekton-resources/kustomization.yaml b/kubernetes/resources/tekton/kustomization.yaml similarity index 78% rename from kubernetes/components/tekton-resources/kustomization.yaml rename to kubernetes/resources/tekton/kustomization.yaml index 01ed69030..3a0d1beb2 100644 --- a/kubernetes/components/tekton-resources/kustomization.yaml +++ b/kubernetes/resources/tekton/kustomization.yaml @@ -1,6 +1,6 @@ --- -apiVersion: kustomize.config.k8s.io/v1alpha1 -kind: Component +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization resources: - cosign-verify.yaml diff --git a/kubernetes/components/tekton-resources/nix2container-copyto.yaml b/kubernetes/resources/tekton/nix2container-copyto.yaml similarity index 100% rename from kubernetes/components/tekton-resources/nix2container-copyto.yaml rename to kubernetes/resources/tekton/nix2container-copyto.yaml diff --git a/kubernetes/components/tekton-resources/nix2container-image-info.yaml b/kubernetes/resources/tekton/nix2container-image-info.yaml similarity index 100% rename from kubernetes/components/tekton-resources/nix2container-image-info.yaml rename to kubernetes/resources/tekton/nix2container-image-info.yaml diff --git a/kubernetes/components/tekton-resources/rebuild-nativelink.yaml b/kubernetes/resources/tekton/rebuild-nativelink.yaml similarity index 94% rename from kubernetes/components/tekton-resources/rebuild-nativelink.yaml rename to kubernetes/resources/tekton/rebuild-nativelink.yaml index f1435bdaa..a007954bf 100644 --- a/kubernetes/components/tekton-resources/rebuild-nativelink.yaml +++ b/kubernetes/resources/tekton/rebuild-nativelink.yaml @@ -26,6 +26,13 @@ spec: image. - "github:/?ref=pull//head#" to use an image from a pull request of a repository on GitHub. + - name: configMapName + type: string + description: | + The name of the ConfigMap that we write the image tag to. + + Other parts of the cluster may refer to this configmap to get the + pre-evaluated tag of the image. - name: registry type: string description: | @@ -172,6 +179,8 @@ spec: value: "$(tasks.get-image-info.results.imageName)" - name: imageTag value: "$(tasks.get-image-info.results.imageTag)" + - name: configMapName + value: "$(params.configMapName)" runAfter: - copy-verified-prebuilt-image - copy-nix-built-image diff --git a/kubernetes/components/tekton-resources/skopeo-check-hashlocked-url.yaml b/kubernetes/resources/tekton/skopeo-check-hashlocked-url.yaml similarity index 100% rename from kubernetes/components/tekton-resources/skopeo-check-hashlocked-url.yaml rename to kubernetes/resources/tekton/skopeo-check-hashlocked-url.yaml diff --git a/kubernetes/components/tekton-resources/skopeo-copy.yaml b/kubernetes/resources/tekton/skopeo-copy.yaml similarity index 100% rename from kubernetes/components/tekton-resources/skopeo-copy.yaml rename to kubernetes/resources/tekton/skopeo-copy.yaml diff --git a/kubernetes/components/tekton-resources/trigger.yaml b/kubernetes/resources/tekton/trigger.yaml similarity index 90% rename from kubernetes/components/tekton-resources/trigger.yaml rename to kubernetes/resources/tekton/trigger.yaml index 9fa64dd04..97d26fedd 100644 --- a/kubernetes/components/tekton-resources/trigger.yaml +++ b/kubernetes/resources/tekton/trigger.yaml @@ -53,6 +53,12 @@ spec: image. - "github:/?ref=pull//head#" to use an image from a pull request of a repository on GitHub. + - name: configMapName + description: | + The name of the ConfigMap that we write the image tag to. + + Other parts of the cluster may refer to this configmap to get the + pre-evaluated tag of the image. resourcetemplates: - apiVersion: tekton.dev/v1beta1 kind: PipelineRun @@ -76,6 +82,8 @@ spec: params: - name: flakeOutput value: "$(tt.params.flakeOutput)" + - name: configMapName + value: "$(tt.params.configMapName)" - name: registry value: "kind-registry:5000" - name: destTLSverify @@ -91,6 +99,8 @@ spec: params: - name: flakeOutput value: "$(body.metadata.flakeOutput)" + - name: configMapName + value: "$(body.metadata.configMapName)" --- apiVersion: v1 kind: ServiceAccount diff --git a/kubernetes/components/tekton-resources/update-image-tags.yaml b/kubernetes/resources/tekton/update-image-tags.yaml similarity index 94% rename from kubernetes/components/tekton-resources/update-image-tags.yaml rename to kubernetes/resources/tekton/update-image-tags.yaml index 7922686e9..75c2ebf28 100644 --- a/kubernetes/components/tekton-resources/update-image-tags.yaml +++ b/kubernetes/resources/tekton/update-image-tags.yaml @@ -14,12 +14,13 @@ spec: params: - name: imageName - name: imageTag + - name: configMapName steps: - name: update-configmap image: bitnami/kubectl env: - name: CM_NAME - value: "nativelink-image-tags" + value: "$(params.configMapName)" script: | #!/bin/bash set -e diff --git a/kubernetes/workers/lre-cc/kustomization.yaml b/kubernetes/workers/lre-cc/kustomization.yaml new file mode 100644 index 000000000..a73b9c052 --- /dev/null +++ b/kubernetes/workers/lre-cc/kustomization.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -lre-cc + +components: +- ../../components/worker + +patches: +- path: worker-lre-cc.yaml + target: + kind: Deployment + name: nativelink-worker + +images: +- name: nativelink-worker + newName: nativelink-worker-lre-cc diff --git a/kubernetes/overlays/lre/worker-lre-cc.yaml b/kubernetes/workers/lre-cc/worker-lre-cc.yaml similarity index 88% rename from kubernetes/overlays/lre/worker-lre-cc.yaml rename to kubernetes/workers/lre-cc/worker-lre-cc.yaml index 46ba6b646..01356fabf 100644 --- a/kubernetes/overlays/lre/worker-lre-cc.yaml +++ b/kubernetes/workers/lre-cc/worker-lre-cc.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: nativelink-worker + name: nativelink-worker-lre-cc spec: replicas: 1 template: @@ -23,9 +23,11 @@ spec: # `nativelink-worker-lre-cc:` which is a # completely separate extension of the `lre-cc` base image. args: + # TODO(aaronmondal): This shouldn't be evaluating the worker image + # tag, but the toolchain tag. - | git config --global --add safe.directory "*" - NATIVELINK_WORKER_PLATFORM=docker://lre-cc:$(nix eval /mnt/src_root#lre-cc.imageTag --raw) && + NATIVELINK_WORKER_PLATFORM=docker://lre-cc:$(nix eval /mnt/src_root#nativelink-worker-lre-cc.imageTag --raw) && printf '#!/bin/sh\nexport NATIVELINK_WORKER_PLATFORM=%s\nexec "$@"' "$NATIVELINK_WORKER_PLATFORM" > /entrypoint/entrypoint.sh && chmod +x /entrypoint/entrypoint.sh volumeMounts: diff --git a/kubernetes/workers/lre-rs/kustomization.yaml b/kubernetes/workers/lre-rs/kustomization.yaml new file mode 100644 index 000000000..38d7568cf --- /dev/null +++ b/kubernetes/workers/lre-rs/kustomization.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -lre-rs + +components: +- ../../components/worker + +patches: +- path: worker-lre-rs.yaml + target: + kind: Deployment + name: nativelink-worker + +images: +- name: nativelink-worker + newName: nativelink-worker-lre-rs diff --git a/kubernetes/workers/lre-rs/worker-lre-rs.yaml b/kubernetes/workers/lre-rs/worker-lre-rs.yaml new file mode 100644 index 000000000..d5fa10c06 --- /dev/null +++ b/kubernetes/workers/lre-rs/worker-lre-rs.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nativelink-worker-lre-rs +spec: + replicas: 1 + template: + spec: + initContainers: + - name: setup-entrypoint + image: nixpkgs/nix-flakes:latest + command: ["/bin/sh", "-c"] + # The kind setup mounts the nativelink repository into the kind nodes + # at `/mnt/src_root`. This ensures that the tags between the worker + # configs and bazel toolchains match when this setup is run in CI. + # + # WARNING: The platform is *not* necessarily the container that is + # actually deployed here. The generator container in this example was + # `rbe-autogen-lre-cc:` and the platform was modified + # after the fact to be `lre-cc:`. The deployed container + # we use as worker is + # `nativelink-worker-lre-cc:` which is a + # completely separate extension of the `lre-cc` base image. + args: + - | + git config --global --add safe.directory "*" + NATIVELINK_WORKER_LRE_RS=$(nix eval /mnt/src_root#nativelink-worker-lre-rs.imageTag --raw) && + printf '#!/bin/sh\nexport NATIVELINK_WORKER_LRE_RS=%s\nexec "$@"' "$NATIVELINK_WORKER_LRE_RS" > /entrypoint/entrypoint.sh && + chmod +x /entrypoint/entrypoint.sh + volumeMounts: + - name: entrypoint + mountPath: /entrypoint + - name: mnt + mountPath: /mnt + containers: + - name: nativelink-worker + volumeMounts: + - name: entrypoint + mountPath: /entrypoint + command: ["/entrypoint/entrypoint.sh"] + args: ["/shared/nativelink", "/worker.json"] + volumes: + - name: entrypoint + emptyDir: {} + - name: mnt + hostPath: + path: /mnt diff --git a/kubernetes/workers/overlays/lre-cc-standalone/kustomization.yaml b/kubernetes/workers/overlays/lre-cc-standalone/kustomization.yaml new file mode 100644 index 000000000..3a76f10c8 --- /dev/null +++ b/kubernetes/workers/overlays/lre-cc-standalone/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../resources/worker-init + - ../../resources/lre-cc diff --git a/kubernetes/workers/overlays/lre-rs-standalone/kustomization.yaml b/kubernetes/workers/overlays/lre-rs-standalone/kustomization.yaml new file mode 100644 index 000000000..2b0327151 --- /dev/null +++ b/kubernetes/workers/overlays/lre-rs-standalone/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../resources/worker-init + - ../../resources/lre-rs diff --git a/kubernetes/workers/overlays/siso-chromium-standalone/kustomization.yaml b/kubernetes/workers/overlays/siso-chromium-standalone/kustomization.yaml new file mode 100644 index 000000000..310311729 --- /dev/null +++ b/kubernetes/workers/overlays/siso-chromium-standalone/kustomization.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -standalone + +resources: + - ../../resources/worker-init + - ../../resources/siso-chromium diff --git a/kubernetes/workers/resources/lre-cc/kustomization.yaml b/kubernetes/workers/resources/lre-cc/kustomization.yaml new file mode 100644 index 000000000..f96708225 --- /dev/null +++ b/kubernetes/workers/resources/lre-cc/kustomization.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -lre-cc + +resources: + - ../../../resources/alert-kustomization + +components: + - ../../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-lre-cc + - op: add + path: /spec/nameSuffix + value: -lre-cc + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/workers/lre-cc + - op: add + path: /spec/postBuild/substituteFrom/1 + value: + kind: ConfigMap + name: nativelink-worker-init + - op: add + path: /spec/images + value: + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker-lre-cc + newName: localhost:5001/nativelink-worker-lre-cc + newTag: ${NATIVELINK_WORKER_LRE_CC_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/workers/resources/lre-rs/kustomization.yaml b/kubernetes/workers/resources/lre-rs/kustomization.yaml new file mode 100644 index 000000000..d0a46d696 --- /dev/null +++ b/kubernetes/workers/resources/lre-rs/kustomization.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -lre-rs + +resources: + - ../../../resources/alert-kustomization + +components: + - ../../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-lre-rs + - op: add + path: /spec/nameSuffix + value: -lre-rs + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/workers/lre-rs + - op: add + path: /spec/postBuild/substituteFrom/1 + value: + kind: ConfigMap + name: nativelink-worker-init + - op: add + path: /spec/images + value: + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker-lre-rs + newName: localhost:5001/nativelink-worker-lre-rs + newTag: ${NATIVELINK_WORKER_LRE_RS_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/workers/resources/siso-chromium/kustomization.yaml b/kubernetes/workers/resources/siso-chromium/kustomization.yaml new file mode 100644 index 000000000..53b83f768 --- /dev/null +++ b/kubernetes/workers/resources/siso-chromium/kustomization.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -siso-chromium + +resources: + - ../../../resources/alert-kustomization + +components: + - ../../../components/kustomization + +replacements: + - source: + kind: Kustomization + name: nativelink + fieldPath: metadata.name + targets: + - select: + kind: Kustomization + name: nativelink-alert + fieldPaths: + - spec.postBuild.substitute.PLACEHOLDER_CONFIG_MAP_NAME + - select: + kind: Kustomization + name: nativelink + fieldPaths: + - spec.postBuild.substituteFrom.0.name + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium + - op: add + path: /spec/nameSuffix + value: -siso-chromium + target: + kind: Kustomization + name: nativelink-alert +- patch: |- + - op: replace + path: /spec/path + value: ./kubernetes/workers/siso-chromium + - op: add + path: /spec/postBuild/substituteFrom/1 + value: + kind: ConfigMap + name: nativelink-worker-init + - op: add + path: /spec/images + value: + - name: nativelink-worker-init + newName: localhost:5001/nativelink-worker-init + newTag: ${NATIVELINK_WORKER_INIT_TAG} + - name: nativelink-worker-siso-chromium + newName: localhost:5001/nativelink-worker-siso-chromium + newTag: ${NATIVELINK_WORKER_SISO_CHROMIUM_TAG} + target: + kind: Kustomization + name: nativelink diff --git a/kubernetes/workers/resources/worker-init/kustomization.yaml b/kubernetes/workers/resources/worker-init/kustomization.yaml new file mode 100644 index 000000000..ee4559366 --- /dev/null +++ b/kubernetes/workers/resources/worker-init/kustomization.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -worker-init + +resources: + - ../../../resources/alert-kustomization + +patches: +- patch: |- + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT + value: github:TraceMachina/nativelink#nativelink-worker-init + - op: replace + path: /spec/postBuild/substitute/PLACEHOLDER_CONFIG_MAP_NAME + value: nativelink-worker-init + - op: add + path: /spec/nameSuffix + value: -worker-init + target: + kind: Kustomization + name: nativelink-alert diff --git a/kubernetes/workers/siso-chromium/kustomization.yaml b/kubernetes/workers/siso-chromium/kustomization.yaml new file mode 100644 index 000000000..babad5ade --- /dev/null +++ b/kubernetes/workers/siso-chromium/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -siso-chromium + +components: +- ../../components/worker + +images: +- name: nativelink-worker + newName: nativelink-worker-siso-chromium diff --git a/local-remote-execution/MODULE.bazel b/local-remote-execution/MODULE.bazel index fed578399..c0d1dcbb9 100644 --- a/local-remote-execution/MODULE.bazel +++ b/local-remote-execution/MODULE.bazel @@ -14,3 +14,18 @@ bazel_dep(name = "rules_cc", version = "0.0.17") # Use the starlark implementation of Java rules instead of the builtin ones. bazel_dep(name = "rules_java", version = "8.5.1") +bazel_dep(name = "rules_rust", version = "0.54.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") + +lre_rs = use_extension("//lre-rs:extension.bzl", "lre_rs") +use_repo( + lre_rs, + "lre-rs-nightly-aarch64-darwin", + "lre-rs-nightly-aarch64-linux", + "lre-rs-nightly-x86_64-darwin", + "lre-rs-nightly-x86_64-linux", + "lre-rs-stable-aarch64-darwin", + "lre-rs-stable-aarch64-linux", + "lre-rs-stable-x86_64-darwin", + "lre-rs-stable-x86_64-linux", +) diff --git a/local-remote-execution/examples/BUILD.bazel b/local-remote-execution/examples/BUILD.bazel index b7efedad8..5b0f4cea3 100644 --- a/local-remote-execution/examples/BUILD.bazel +++ b/local-remote-execution/examples/BUILD.bazel @@ -5,6 +5,7 @@ cc_binary( name = "hello_lre", srcs = ["hello.cpp"], copts = ["--verbose"], + linkopts = ["-Wl,--verbose"], target_compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", diff --git a/local-remote-execution/flake-module.nix b/local-remote-execution/flake-module.nix index 6fdda93de..ae9228815 100644 --- a/local-remote-execution/flake-module.nix +++ b/local-remote-execution/flake-module.nix @@ -45,4 +45,13 @@ } ); }; + + config = { + perSystem = {pkgs, ...}: { + packages = { + inherit (pkgs.lre) stable-rust nightly-rust; + rbe-autogen-lre-cc = pkgs.rbe-autogen pkgs.lre.lre-cc.image; + }; + }; + }; } diff --git a/local-remote-execution/libc/BUILD.bazel b/local-remote-execution/libc/BUILD.bazel new file mode 100644 index 000000000..766554cb1 --- /dev/null +++ b/local-remote-execution/libc/BUILD.bazel @@ -0,0 +1,13 @@ +constraint_setting(name = "libc") + +constraint_value( + name = "glibc", + constraint_setting = ":libc", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "musl", + constraint_setting = ":libc", + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-cc.nix b/local-remote-execution/lre-cc.nix deleted file mode 100644 index a74ed385b..000000000 --- a/local-remote-execution/lre-cc.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ - buildImage, - lre, - lib, - coreutils, - findutils, - gnutar, -}: let - # This environment is shared between toolchain autogen images and the final - # toolchain image. - Env = [ - # Add all tooling here so that the generated toolchains use `/nix/store/*` - # paths instead of `/bin` or `/usr/bin`. This way we're guaranteed to use - # binary identical toolchains during local and remote execution. - ("PATH=" - + (lib.strings.concatStringsSep ":" [ - "${lre.stdenv.cc.bintools}/bin" - "${lre.clang}/bin" - "${lre.stdenv}/bin" - "${coreutils}/bin" - "${findutils}/bin" - "${gnutar}/bin" - ])) - - "CC=${lre.clang}/bin/customClang" - ]; -in - buildImage { - name = "lre-cc"; - maxLayers = 100; - config = {inherit Env;}; - # Attached for passthrough to rbe-configs-gen. - meta = {inherit Env;}; - - # Don't set a tag here so that the image is tagged by its derivation hash. - # tag = null; - } diff --git a/local-remote-execution/lre-rs/BUILD.bazel b/local-remote-execution/lre-rs/BUILD.bazel new file mode 100644 index 000000000..ba0cd85de --- /dev/null +++ b/local-remote-execution/lre-rs/BUILD.bazel @@ -0,0 +1,242 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load( + "@rules_rust//rust:toolchain.bzl", + "rust_toolchain", + "rustfmt_toolchain", +) + +string_flag( + name = "channel", + build_setting_default = "stable", + values = [ + "stable", + "nightly", + ], + visibility = ["//visibility:public"], +) + +config_setting( + name = "stable", + flag_values = {":channel": "stable"}, +) + +config_setting( + name = "nightly", + flag_values = {":channel": "nightly"}, +) + +string_flag( + name = "rustfmt-channel", + # It's a common use case (and matching with rules_rust's default behavior) + # to use nightly rustfmt with a stable rust toolchain, so we default to + # nightly here. + build_setting_default = "nightly", + values = [ + "stable", + "nightly", + ], + visibility = ["//visibility:public"], +) + +config_setting( + name = "rustfmt-stable", + flag_values = {":rustfmt-channel": "stable"}, +) + +config_setting( + name = "rustfmt-nightly", + flag_values = {":rustfmt-channel": "nightly"}, +) + +[ + rust_toolchain( + name = "rust-{}-{}_impl".format(channel, nix_system), + allocator_library = select({ + "@local-remote-execution//libc:musl": None, + "//conditions:default": "@rules_rust//ffi/cc/allocator_library", + }), + binary_ext = "", + cargo = "@lre-rs-{}-{}//:cargo".format(channel, nix_system), + cargo_clippy = "@lre-rs-{}-{}//:cargo-clippy".format(channel, nix_system), + clippy_driver = "@lre-rs-{}-{}//:clippy-driver".format(channel, nix_system), + debug_info = { + "dbg": "2", + "fastbuild": "0", + "opt": "0", + }, + default_edition = "2021", + dylib_ext = select({ + "@platforms//os:linux": ".so", + "@platforms//os:macos": ".dylib", + "//conditions:default": "@platforms//:incompatible", + }), + env = {}, + exec_triple = { + "x86_64-linux": "x86_64-unknown-linux-gnu", + "aarch64-linux": "aarch64-unknown-linux-gnu", + "x86_64-darwin": "x86_64-apple-darwin", + "aarch64-darwin": "aarch64-apple-darwin", + }[nix_system], + + # TODO(aaronmondal): Make these easily configurable. + extra_exec_rustc_flags = [], + extra_rustc_flags = [], + extra_rustc_flags_for_crate_types = {}, + global_allocator_library = select({ + "@local-remote-execution//libc:musl": None, + "//conditions:default": "@rules_rust//ffi/cc/global_allocator_library", + }), + + # TODO(aaronmondal): Implement these. + # llvm_cov, + # llvm_profdata, + # llvm_tools, + opt_level = { + "dbg": "0", + "fastbuild": "0", + "opt": "3", + }, + per_crate_rustc_flags = [], + rust_doc = "@lre-rs-{}-{}//:rustdoc".format(channel, nix_system), + rust_std = "@lre-rs-{}-{}//:rust_std".format(channel, nix_system), + rustc = "@lre-rs-{}-{}//:rustc".format(channel, nix_system), + rustc_lib = "@lre-rs-{}-{}//:rustc_lib".format(channel, nix_system), + rustfmt = None, # Rustfmt actions use the rustfmt-toolchain. + staticlib_ext = ".a", + stdlib_linkflags = select( + { + "@local-remote-execution//libc:glibc": [ + "-fuse-ld=mold", + "-lpthread", + "-ldl", + ], + "@local-remote-execution//libc:musl": [ + "-fuse-ld=mold", + ], + "@platforms//os:macos": [], + }, + no_match_error = "Linking for unset libc on linux isn't implemented yet.", + ), + strip_level = { + "dbg": "none", + "fastbuild": "none", + "opt": "debuginfo", + }, + target_json = "", + target_triple = select( + { + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": "aarch64-unknown-linux-gnu", + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": "aarch64-unknown-linux-musl", + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": "x86_64-unknown-linux-gnu", + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": "x86_64-unknown-linux-musl", + } | ({ + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": "x86_64-apple-darwin", + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": "aarch64-apple-darwin", + } if nix_system in [ + "aarch64-darwin", + "x86_64-darwin", + ] else {}), + no_match_error = "Incompatible target triple supplied to lre-rs-toolchain running on {}.".format(nix_system), + ), + ) + for channel in [ + "stable", + "nightly", + ] + for nix_system in [ + "x86_64-linux", + "aarch64-linux", + "x86_64-darwin", + "aarch64-darwin", + ] +] + +[ + toolchain( + name = "rust-{}-{}".format(arch, os), + exec_compatible_with = [ + "@platforms//os:{}".format(os if os == "linux" else "macos"), + "@platforms//cpu:{}".format(arch), + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = + # Linux toolchains can only target other linux systems. + # Darwin toolchains can target everything. + ["@platforms//os:linux"] if os == "linux" else [], + toolchain = select({ + ":stable": ":rust-stable-{}-{}_impl".format(arch, os), + ":nightly": ":rust-nightly-{}-{}_impl".format(arch, os), + }), + toolchain_type = "@rules_rust//rust:toolchain_type", + ) + for arch in [ + "aarch64", + "x86_64", + ] + for os in [ + "linux", + "darwin", + ] +] + +[ + rustfmt_toolchain( + name = "rustfmt-{}-{}_impl".format(channel, nix_system), + rustc = "@lre-rs-{}-{}//:rustc".format(channel, nix_system), + rustc_lib = "@lre-rs-{}-{}//:rustc_lib".format(channel, nix_system), + rustfmt = "@lre-rs-{}-{}//:rustfmt".format(channel, nix_system), + ) + for nix_system in [ + "x86_64-linux", + "aarch64-linux", + "x86_64-darwin", + "aarch64-darwin", + ] + for channel in [ + "stable", + "nightly", + ] +] + +# Limit rustfmt to exec == target until we find a good reason to do otherwise. +[ + toolchain( + name = "rustfmt-{}-{}".format(arch, os), + exec_compatible_with = [ + "@platforms//os:{}".format(os if os == "linux" else "macos"), + "@platforms//cpu:{}".format(arch), + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = [ + "@platforms//os:{}".format(os if os == "linux" else "macos"), + "@platforms//cpu:{}".format(arch), + ], + toolchain = select({ + ":rustfmt-stable": ":rustfmt-stable-{}-{}_impl".format(arch, os), + ":rustfmt-nightly": ":rustfmt-nightly-{}-{}_impl".format(arch, os), + }), + toolchain_type = "@rules_rust//rust/rustfmt:toolchain_type", + ) + for arch in [ + "aarch64", + "x86_64", + ] + for os in [ + "darwin", + "linux", + ] +] diff --git a/local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel b/local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel new file mode 100644 index 000000000..d72b70d44 --- /dev/null +++ b/local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel @@ -0,0 +1,125 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on aarch64-darwin.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/aarch64-apple-darwin/bin/rust-lld", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/x86_64-apple-darwin/bin/rust-lld", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "lib/rustlib/aarch64-apple-darwin/lib/*.rlib", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + "lib/rustlib/aarch64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "lib/rustlib/x86_64-apple-darwin/lib/*.rlib", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + "lib/rustlib/x86_64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel b/local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel new file mode 100644 index 000000000..7a5474a64 --- /dev/null +++ b/local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel @@ -0,0 +1,101 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on aarch64-linux.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs/extension.bzl b/local-remote-execution/lre-rs/extension.bzl new file mode 100644 index 000000000..f40abda65 --- /dev/null +++ b/local-remote-execution/lre-rs/extension.bzl @@ -0,0 +1,71 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Module extension to register different lre-rs tool repositories.""" + +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +# TODO(aaronmondal): Using module extensions here isn't optimal as it doesn't +# allow overriding tools via environment variables. +# Unfortunately rules_rust's rust_toolchain currently +# requires tools to be declared via labels to filegroups, +# so we need the new_local_repository approach here. +# Add support for raw strings to rules_rust upstream so +# that we can remove this module extension entirely. + +def _lre_rs_impl(_mctx): + new_local_repository( + name = "lre-rs-stable-aarch64-darwin", + build_file = "@local-remote-execution//lre-rs:aarch64-darwin.BUILD.bazel", + path = "/nix/store/59m1dvvnr0imcwdd0xiqaljis9ybvy24-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-aarch64-darwin", + build_file = "@local-remote-execution//lre-rs:aarch64-darwin.BUILD.bazel", + path = "/nix/store/pjd6k6zj8blkxr551bkpz0xq3ycj9yhn-rust-default-1.85.0-nightly-2024-11-23", + ) + new_local_repository( + name = "lre-rs-stable-aarch64-linux", + build_file = "@local-remote-execution//lre-rs:aarch64-linux.BUILD.bazel", + path = "/nix/store/pyr87frpmb04bh3s5sf7vszz9855w603-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-aarch64-linux", + build_file = "@local-remote-execution//lre-rs:aarch64-linux.BUILD.bazel", + path = "/nix/store/afdl7canh5p6krqzw5vm1g4kihhqfvsr-rust-default-1.85.0-nightly-2024-11-23", + ) + new_local_repository( + name = "lre-rs-stable-x86_64-darwin", + build_file = "@local-remote-execution//lre-rs:x86_64-darwin.BUILD.bazel", + path = "/nix/store/x030cxhzj00bw41yaqiy2ihjj1mgi6mr-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-x86_64-darwin", + build_file = "@local-remote-execution//lre-rs:x86_64-darwin.BUILD.bazel", + path = "/nix/store/g1irmsn6x1gndkjisal978jfs5s0dqb2-rust-default-1.85.0-nightly-2024-11-23", + ) + new_local_repository( + name = "lre-rs-stable-x86_64-linux", + build_file = "@local-remote-execution//lre-rs:x86_64-linux.BUILD.bazel", + path = "/nix/store/kl18sr2a45vyyqv39clrjclmadacy9mc-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-x86_64-linux", + build_file = "@local-remote-execution//lre-rs:x86_64-linux.BUILD.bazel", + path = "/nix/store/902a168nsh4vha014j538abaji3lpwa2-rust-default-1.85.0-nightly-2024-11-23", + ) + +lre_rs = module_extension(implementation = _lre_rs_impl) diff --git a/local-remote-execution/lre-rs/platforms/BUILD.bazel b/local-remote-execution/lre-rs/platforms/BUILD.bazel new file mode 100644 index 000000000..3a926afcd --- /dev/null +++ b/local-remote-execution/lre-rs/platforms/BUILD.bazel @@ -0,0 +1,78 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +# Some toolchains, like the darwin ones, don't have an actual container image. +# We still map them to the output of the tag of the corresponding theoretical +# worker image so that bare metal metal workers can advertise exact +# exec_properties to schedulers. + +platform( + name = "aarch64-apple-darwin", + exec_properties = { + # nix eval .#packages.aarch64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": "xbwngvrnr7ygazcr20a3mddsx05l3nxn", + }, + parents = ["@local-remote-execution//nix-system:aarch64-darwin"], +) + +platform( + name = "aarch64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "j8gd39m51074dgm3d82bm68dda8lbg62", + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], +) + +platform( + name = "aarch64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "j8gd39m51074dgm3d82bm68dda8lbg62", + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], +) + +platform( + name = "x86_64-apple-darwin", + exec_properties = { + # nix eval .#packages.x86_64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": "ywhklqs9l7lz59c1l2fzqp2nzza1c75q", + }, + parents = ["@local-remote-execution//nix-system:x86_64-darwin"], +) + +platform( + name = "x86_64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "phg85ss1hq298ixjn8iqnbrhh5x5x384", + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], +) + +platform( + name = "x86_64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "phg85ss1hq298ixjn8iqnbrhh5x5x384", + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], +) diff --git a/local-remote-execution/lre-rs/triple/BUILD.bazel b/local-remote-execution/lre-rs/triple/BUILD.bazel new file mode 100644 index 000000000..3a60ceade --- /dev/null +++ b/local-remote-execution/lre-rs/triple/BUILD.bazel @@ -0,0 +1,69 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These config settings correspond to rust triples. + +load("@bazel_skylib//lib:selects.bzl", "selects") + +selects.config_setting_group( + name = "aarch64-apple-darwin", + match_all = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], +) + +selects.config_setting_group( + name = "aarch64-unknown-linux-gnu", + match_all = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@local-remote-execution//libc:glibc", + ], +) + +selects.config_setting_group( + name = "aarch64-unknown-linux-musl", + match_all = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@local-remote-execution//libc:musl", + ], +) + +selects.config_setting_group( + name = "x86_64-apple-darwin", + match_all = [ + "@platforms//cpu:x86_64", + "@platforms//os:macos", + ], +) + +selects.config_setting_group( + name = "x86_64-unknown-linux-gnu", + match_all = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + "@local-remote-execution//libc:glibc", + ], +) + +selects.config_setting_group( + name = "x86_64-unknown-linux-musl", + match_all = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + "@local-remote-execution//libc:musl", + ], +) diff --git a/local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel b/local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel new file mode 100644 index 000000000..016833c6b --- /dev/null +++ b/local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel @@ -0,0 +1,125 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on x86_64-darwin.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/aarch64-apple-darwin/bin/rust-lld", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/x86_64-apple-darwin/bin/rust-lld", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "lib/rustlib/aarch64-apple-darwin/lib/*.rlib", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + "lib/rustlib/aarch64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "lib/rustlib/x86_64-apple-darwin/lib/*.rlib", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + "lib/rustlib/x86_64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel b/local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel new file mode 100644 index 000000000..000925c3e --- /dev/null +++ b/local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel @@ -0,0 +1,101 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on x86_64-linux.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/modules/lre.nix b/local-remote-execution/modules/lre.nix index 4d3f08582..3e9936ac4 100644 --- a/local-remote-execution/modules/lre.nix +++ b/local-remote-execution/modules/lre.nix @@ -4,6 +4,30 @@ pkgs, ... }: let + nixExecToRustExec = p: let + inherit (p.stdenv.buildPlatform) system; + in + { + "aarch64-darwin" = "aarch64-apple-darwin"; + "aarch64-linux" = "aarch64-unknown-linux-gnu"; + "x86_64-darwin" = "x86_64-apple-darwin"; + "x86_64-linux" = "x86_64-unknown-linux-gnu"; + } + .${system} + or (throw "Unsupported Nix exec platform: ${system}"); + + nixExecToDefaultRustTarget = p: let + inherit (p.stdenv.targetPlatform) system; + in + { + "aarch64-darwin" = "aarch64-apple-darwin"; + "aarch64-linux" = "aarch64-unknown-linux-musl"; + "x86_64-darwin" = "x86_64-apple-darwin"; + "x86_64-linux" = "x86_64-unknown-linux-musl"; + } + .${system} + or (throw "Unsupported Nix target platform: ${system}"); + # These flags cause a Bazel build to use LRE toolchains regardless of whether # the build is running in local or remote configuration. # @@ -21,27 +45,61 @@ # ''; # }; # ``` - defaultConfig = [ - # TODO(aaronmondal): Remove after resolution of: - # https://github.com/bazelbuild/bazel/issues/19714#issuecomment-1745604978 - "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1" - - # TODO(aaronmondal): Remove after resolution of: - # https://github.com/bazelbuild/bazel/issues/7254 - "--define=EXECUTOR=remote" - - # Set up the default toolchains. - # TODO(aaronmondal): Implement a mechanism that autogenerates these values - # and generalizes to extensions of the lre-cc base - # toolchain (such as CUDA extensions). - "--extra_execution_platforms=@local-remote-execution//generated-cc/config:platform" - "--extra_toolchains=@local-remote-execution//generated-cc/config:cc-toolchain" - ]; + defaultConfig = + [ + # Global configuration. + + # TODO(aaronmondal): Remove after resolution of: + # https://github.com/bazelbuild/bazel/issues/7254 + "--define=EXECUTOR=remote" + ] + # C++. + # TODO(aaronmondal): At the moment lre-cc only supports x86_64-linux. + # Extend this to more nix systems. + # See: https://github.com/bazelbuild/bazel/issues/19714#issuecomment-1745604978 + ++ lib.optionals pkgs.stdenv.isLinux [ + "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1" + + # TODO(aaronmondal): Reimplement rbe-configs-gen for C++ so that we can + # support more execution and target platforms here. + + # Explicitly duplicate the host as an execution platform. This way local + # execution is treated the same as remote execution. + "--extra_execution_platforms=@local-remote-execution//generated-cc/config:platform" + # The C++ toolchain running on the execution platform. + "--extra_toolchains=@local-remote-execution//generated-cc/config:cc-toolchain" + + # TODO(aaronmondal): Support different target platforms in lre-cc and add + # `--platforms` settings here. + ] + # Rust. + ++ [ + # Explicitly duplicate the host as an execution platform. This way local + # execution is treated the same as remote execution. + # + # When using remote executors that differ from the host this needs to be + # manually extended via the `--extra_execution_platforms` flag. + "--extra_execution_platforms=@local-remote-execution//lre-rs/platforms:${nixExecToRustExec pkgs}" + + # The rust toolchains executing on the execution platform. We default to the + # platforms corresponding to the host. When using remote executors that + # differ from the platform corresponding to the host this should be extended + # via manual `--extra_toolchains` arguments. + "--extra_toolchains=@local-remote-execution//lre-rs:rust-${pkgs.system}" + "--extra_toolchains=@local-remote-execution//lre-rs:rustfmt-${pkgs.system}" + + # Defaults for rust target platforms. This is a convenience setting that + # may be overridden by manual `--platforms` arguments. + # + # TODO(aaronmondal): At the moment these platforms are "rust-specific". + # Generalize this to all languages. + "--platforms=@local-remote-execution//lre-rs/platforms:${nixExecToDefaultRustTarget pkgs}" + ]; maybeEnv = if config.Env == [] then ["#" "# WARNING: No environment set. LRE will not work locally."] - else ["#"] ++ (map (x: "# " + x) config.Env); + else ["#"] ++ (map (x: "# " + x) (lib.lists.unique config.Env)); # If the `local-remote-execution.settings.prefix` is set to a nonempty string, # prefix the Bazel build commands with that string. This will disable LRE diff --git a/local-remote-execution/nix-system/BUILD.bazel b/local-remote-execution/nix-system/BUILD.bazel new file mode 100644 index 000000000..bf1c2b2c8 --- /dev/null +++ b/local-remote-execution/nix-system/BUILD.bazel @@ -0,0 +1,87 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These platforms correspond to nix systems. + +# The exec_properties correspond to kubernetes properties that help RBE systems +# to determine on which nodes in a cluster the platform could realistically be +# deployed to. For instance, you'll likely always need to match the CPU and +# you'll always want to run mac platforms on mac nodes. However, you can deploy +# linux containers onto mac systems as long as the cpu architecture matches, so +# we don't constrain "kubernetes.io/os" for the linux platforms. + +# If these defaults don't work for you you can unset exec_properties by +# overriding them with empty strings in child platforms. +# See: https://bazel.build/versions/7.4.0/reference/be/platforms-and-toolchains + +# The sensible values for "kubernetes.io/arch" and "kubernetes.io/os" correspond +# to those of GOOS and GOARCH. +# See: https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63. + +# TODO(aaronmondal): Reevaluate this implementation once Bazel 8 is released +# which adds support for platform-specific flags. + +platform( + name = "aarch64-darwin", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "arm64", + "kubernetes.io/os": "darwin", + }, + visibility = ["//visibility:public"], +) + +platform( + name = "aarch64-linux", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "arm64", + }, + visibility = ["//visibility:public"], +) + +platform( + name = "x86_64-darwin", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:macos", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "amd64", + "kubernetes.io/os": "darwin", + }, + visibility = ["//visibility:public"], +) + +platform( + name = "x86_64-linux", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "amd64", + }, + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/overlays/default.nix b/local-remote-execution/overlays/default.nix index 3595a96fd..0d6f3f3fe 100644 --- a/local-remote-execution/overlays/default.nix +++ b/local-remote-execution/overlays/default.nix @@ -1,12 +1,46 @@ -final: _prev: { - lre = { - stdenv = final.callPackage ./stdenv.nix { - llvmPackages = final.llvmPackages_19; - targetPackages = final; - }; - - clang = final.callPackage ./clang.nix { - inherit (final.lre) stdenv; - }; +{nix2container}: final: _prev: { + inherit (nix2container.packages.${final.system}) nix2container; + + rbe-configs-gen = final.callPackage ./rbe-configs-gen {}; + + rbe-autogen = final.callPackage ./rbe-autogen.nix { + inherit (final.lre) stdenv; }; + + lre = + { + stdenv = final.callPackage ./stdenv.nix { + llvmPackages = final.llvmPackages_19; + targetPackages = final; + }; + + clang = final.callPackage ./clang.nix { + inherit (final.lre) stdenv; + }; + } + // (let + rustConfig = import ./rust-config.nix; + + stableRustFor = p: (rustConfig.mkRust { + execPkgs = p; + channel = "stable"; + }); + + nightlyRustFor = p: (rustConfig.mkRust { + execPkgs = p; + channel = "nightly"; + extensions = ["llvm-tools"]; + }); + + stable-rust = stableRustFor final; + nightly-rust = nightlyRustFor final; + in { + inherit stable-rust nightly-rust stableRustFor nightlyRustFor; + + lre-rs = final.callPackage ./lre-rs.nix { + inherit (rustConfig) nixSystemToRustTargets; + }; + + lre-cc = final.callPackage ./lre-cc.nix {}; + }); } diff --git a/local-remote-execution/overlays/lre-cc.nix b/local-remote-execution/overlays/lre-cc.nix new file mode 100644 index 000000000..8f2ca21b7 --- /dev/null +++ b/local-remote-execution/overlays/lre-cc.nix @@ -0,0 +1,84 @@ +{ + nix2container, + lre, + lib, + coreutils, + findutils, + gnutar, + bazel_7, + writeShellScriptBin, + rbe-configs-gen, +}: let + lre-cc-configs-gen = writeShellScriptBin "lre-cc" '' + set -xeuo pipefail + + SRC_ROOT=$(git rev-parse --show-toplevel)/local-remote-execution + + cd "''${SRC_ROOT}" + + LRE_CC_IMAGE_TAG=${lre.lre-cc.image.imageTag}; + + nix run .#rbe-autogen-lre-cc.copyTo \ + docker-daemon:rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} -L + + ${rbe-configs-gen}/bin/rbe_configs_gen \ + --toolchain_container=rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} \ + --exec_os=linux \ + --target_os=linux \ + --bazel_version=${bazel_7.version} \ + --output_src_root=''${SRC_ROOT} \ + --output_config_path=generated-cc \ + --generate_java_configs=false \ + --generate_cpp_configs=true \ + --bazel_path=${bazel_7}/bin/bazel \ + --cpp_env_json=cpp_env.json + + # The rbe_configs_gen tool automatically sets the exec_properties of the + # generated platform to the generator container name and tag. For efficiency + # reasons the actual deployment won't be the same as this generator + # container, so we modify this in the generated configuration. + sed -i \ + 's|rbe-autogen-lre-cc|lre-cc|g' \ + ''${SRC_ROOT}/generated-cc/config/BUILD + + chmod 644 \ + ''${SRC_ROOT}/generated-cc/LICENSE \ + ''${SRC_ROOT}/generated-cc/config/BUILD \ + + pre-commit run -a + ''; + + # TODO(aaronmondal): Move the generator logic into this packageset. + # This environment is shared between toolchain autogen images and the final + # toolchain image. + Env = [ + # Add all tooling here so that the generated toolchains use `/nix/store/*` + # paths instead of `/bin` or `/usr/bin`. This way we're guaranteed to use + # binary identical toolchains during local and remote execution. + ("PATH=" + + (lib.strings.concatStringsSep ":" [ + "${lre.stdenv.cc.bintools}/bin" + "${lre.clang}/bin" + "${lre.stdenv}/bin" + "${coreutils}/bin" + "${findutils}/bin" + "${gnutar}/bin" + ])) + + "CC=${lre.clang}/bin/customClang" + ]; + + image = nix2container.buildImage { + name = "lre-cc"; + maxLayers = 100; + config = {inherit Env;}; + # Attached for passthrough to rbe-configs-gen. + meta = {inherit Env;}; + + # Don't set a tag here so that the image is tagged by its derivation hash. + # tag = null; + }; +in { + inherit lre-cc-configs-gen image; + meta = {inherit Env;}; +} diff --git a/local-remote-execution/overlays/lre-rs.nix b/local-remote-execution/overlays/lre-rs.nix new file mode 100644 index 000000000..3c4aa8fbd --- /dev/null +++ b/local-remote-execution/overlays/lre-rs.nix @@ -0,0 +1,248 @@ +{ + lib, + nixSystemToRustTargets, + writeShellScriptBin, + nix2container, + lre, + rust-toolchains ? [lre.stable-rust lre.nightly-rust], +}: let + copyright = '' + # Copyright 2024 The NativeLink Authors. All rights reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # This file is @generated by lre-rs. + ''; + + systems = [ + "aarch64-darwin" + "aarch64-linux" + "x86_64-darwin" + "x86_64-linux" + ]; + channels = [ + "stable" + "nightly" + ]; + + makeRepoDefinition = system: channel: '' + ${" "} new_local_repository( + ${" "} name = "lre-rs-${channel}-${system}", + ${" "} build_file = "@local-remote-execution//lre-rs:${system}.BUILD.bazel", + ${" "} path = "$(nix eval .#packages.${system}.${channel}-rust.outPath --raw)", + ${" "} ) + ''; + + allRepoDefinitions = lib.concatStrings (lib.flatten ( + map (system: map (channel: makeRepoDefinition system channel) channels) systems + )); + + buildFile = system: let + rustTargets = nixSystemToRustTargets.${system}; + + # Generate select entry for a target + mkSelectEntry = target: '' + ${" "} "@local-remote-execution//lre-rs/triple:${target}": glob([ + ${" "} "bin/*.so", + ${" "} "lib/*.so", + ${" "} "lib/rustlib/${target}/codegen-backends/*.so", + ${" "} "lib/rustlib/${target}/bin/rust-lld", + ${" "} "lib/rustlib/${target}/lib/*.so", + ${" "} ]),''; + + # Generate stdlib select entry for a target + mkStdlibSelectEntry = target: + '' + ${" "} "@local-remote-execution//lre-rs/triple:${target}": glob([ + ${" "} "lib/rustlib/${target}/lib/*.rlib", + ${" "} "lib/rustlib/${target}/lib/*.so", + ${" "} "lib/rustlib/${target}/lib/*.a",'' + + (lib.optionalString (builtins.match ".*-musl" target != null) + ''${"\n "} "lib/rustlib/${target}/lib/self-contained/**",'') + + ''${"\n "} ]),''; + + selectEntries = builtins.concatStringsSep "\n" (map mkSelectEntry rustTargets); + stdlibSelectEntries = builtins.concatStringsSep "\n" (map mkStdlibSelectEntry rustTargets); + in '' + ${copyright} + """Tool repository for lre-rs executing on ${system}.""" + + load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + + [ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] + ] + + filegroup( + name = "rustc_lib", + srcs = select({ + ${selectEntries} + }), + visibility = ["//visibility:public"], + ) + + rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + ${stdlibSelectEntries} + }), + visibility = ["//visibility:public"], + )''; + + lre-rs-configs-gen = writeShellScriptBin "lre-rs" '' + set -euo pipefail + + SRC_ROOT=$(git rev-parse --show-toplevel)/local-remote-execution + + cd "''${SRC_ROOT}" + + cat > ''${SRC_ROOT}/lre-rs/extension.bzl << EOF + ${copyright} + """Module extension to register different lre-rs tool repositories.""" + + load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + + # TODO(aaronmondal): Using module extensions here isn't optimal as it doesn't + # allow overriding tools via environment variables. + # Unfortunately rules_rust's rust_toolchain currently + # requires tools to be declared via labels to filegroups, + # so we need the new_local_repository approach here. + # Add support for raw strings to rules_rust upstream so + # that we can remove this module extension entirely. + + def _lre_rs_impl(_mctx): + ${allRepoDefinitions} + lre_rs = module_extension(implementation = _lre_rs_impl) + EOF + + cat > ''${SRC_ROOT}/lre-rs/platforms/BUILD.bazel << EOF + ${copyright} + # Some toolchains, like the darwin ones, don't have an actual container image. + # We still map them to the output of the tag of the corresponding theoretical + # worker image so that bare metal metal workers can advertise exact + # exec_properties to schedulers. + + platform( + name = "aarch64-apple-darwin", + exec_properties = { + # nix eval .#packages.aarch64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.aarch64-darwin.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:aarch64-darwin"], + ) + + platform( + name = "aarch64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], + ) + + platform( + name = "aarch64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], + ) + + platform( + name = "x86_64-apple-darwin", + exec_properties = { + # nix eval .#packages.x86_64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.x86_64-darwin.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:x86_64-darwin"], + ) + + platform( + name = "x86_64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], + ) + + platform( + name = "x86_64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], + ) + EOF + + cat > ''${SRC_ROOT}/lre-rs/aarch64-darwin.BUILD.bazel << EOF + ${buildFile "aarch64-darwin"} + EOF + + cat > ''${SRC_ROOT}/lre-rs/aarch64-linux.BUILD.bazel << EOF + ${buildFile "aarch64-linux"} + EOF + + cat > ''${SRC_ROOT}/lre-rs/x86_64-darwin.BUILD.bazel << EOF + ${buildFile "x86_64-darwin"} + EOF + + cat > ''${SRC_ROOT}/lre-rs/x86_64-linux.BUILD.bazel << EOF + ${buildFile "x86_64-linux"} + EOF''; + + Env = + # Rust requires a functional C++ toolchain. + lre.lre-cc.image.meta.Env + ++ [ + # This causes the rust toolchains to be available under `/nix/store/*` + # paths but not under "generic" paths like `/bin` or `/usr/bin`. + # This way we're guaranteed to use binary identical toolchains during + # local and remote execution. + "RUST=${lib.concatStringsSep ":" rust-toolchains}" + ]; + + image = nix2container.buildImage { + name = "lre-rs"; + maxLayers = 100; + config = {inherit Env;}; + # Passthrough so that other images can reuse the environment. + meta = {inherit Env;}; + + # Don't set a tag here so that the image is tagged by its derivation hash. + # tag = null; + }; +in { + inherit lre-rs-configs-gen image; + meta = {inherit Env;}; +} diff --git a/local-remote-execution/rbe-autogen.nix b/local-remote-execution/overlays/rbe-autogen.nix similarity index 98% rename from local-remote-execution/rbe-autogen.nix rename to local-remote-execution/overlays/rbe-autogen.nix index 9af521466..9332c855f 100644 --- a/local-remote-execution/rbe-autogen.nix +++ b/local-remote-execution/overlays/rbe-autogen.nix @@ -2,7 +2,7 @@ bash, bazel_7, buildEnv, - buildImage, + nix2container, cacert, coreutils, findutils, @@ -96,7 +96,7 @@ }; in image: - buildImage { + nix2container.buildImage { name = "autogen-${image.imageName}"; fromImage = image; maxLayers = 20; diff --git a/local-remote-execution/rbe-configs-gen.nix b/local-remote-execution/overlays/rbe-configs-gen/default.nix similarity index 100% rename from local-remote-execution/rbe-configs-gen.nix rename to local-remote-execution/overlays/rbe-configs-gen/default.nix diff --git a/local-remote-execution/rbe_configs_gen_adjustments.diff b/local-remote-execution/overlays/rbe-configs-gen/rbe_configs_gen_adjustments.diff similarity index 100% rename from local-remote-execution/rbe_configs_gen_adjustments.diff rename to local-remote-execution/overlays/rbe-configs-gen/rbe_configs_gen_adjustments.diff diff --git a/local-remote-execution/overlays/rust-config.nix b/local-remote-execution/overlays/rust-config.nix new file mode 100644 index 000000000..6cb9e69e8 --- /dev/null +++ b/local-remote-execution/overlays/rust-config.nix @@ -0,0 +1,76 @@ +let + defaultStableVersion = "1.82.0"; + defaultNightlyVersion = "2024-11-23"; +in rec { + # This map translates execution platforms to sensible targets that can + # be built on such a platform. For instance, an x86_64-linux execution + # platform can target aarch64-linux and musl targets, but not darwin. + # + # On the Bazel side each key maps to a dedicated toolchain capture the + # cross-compile capabilities on the different exec platforms. + nixSystemToRustTargets = { + "aarch64-darwin" = [ + "aarch64-apple-darwin" + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-apple-darwin" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + "aarch64-linux" = [ + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + "x86_64-darwin" = [ + "aarch64-apple-darwin" + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-apple-darwin" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + "x86_64-linux" = [ + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + }; + + # Changing execPkgs here allows creating custom cross-compilation toolchains: + # + # custom-rust = rustConfig.mkRust { + # execPkgs = pkgs; + # channel = "stable"; + # version = "1.83.0"; # You can override the version as well, but you'll + # # need to run lre-rs afterwards to regenerate the + # # Bazel-side rust toolchains. + # } + # + mkRust = { + execPkgs, + channel, + extensions ? [], + }: let + version = + if channel == "stable" + then defaultStableVersion + else defaultNightlyVersion; + inherit (execPkgs.stdenv.buildPlatform) system; + targets = + nixSystemToRustTargets.${system} + or (throw "Unsupported Nix exec platform: ${system}"); + in + execPkgs.rust-bin.${channel}.${version}.default.override { + inherit extensions targets; + } + // { + # The version identifier generated by rust-bin can't be used by + # `pkgs.rust-bin.${channel}.version`. This passthrough allows using the + # version in classic rust imports like so: + # `pkgs.rust-bin.${pkgs.lre.stable-rust.meta.version;}` + meta = {inherit version;}; + }; +} diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel index 3d36b2998..19b590b5e 100644 --- a/nativelink-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -57,6 +57,10 @@ rust_doc( "--output-format=json", "-Zunstable-options", ], + target_compatible_with = select({ + "@local-remote-execution//lre-rs:nightly": [], + "//conditions:default": ["@platforms//:incompatible"], + }), ) rust_doc_test( diff --git a/tools/generate-toolchains.nix b/tools/generate-toolchains.nix index 190c960c4..1ffaa4898 100644 --- a/tools/generate-toolchains.nix +++ b/tools/generate-toolchains.nix @@ -10,35 +10,6 @@ writeShellScriptBin "generate-toolchains" '' cd "''${SRC_ROOT}" - LRE_CC_IMAGE_TAG=$(nix eval .#lre-cc.imageTag --raw) - - nix run .#rbe-autogen-lre-cc.copyTo \ - docker-daemon:rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} -L - - ${rbe-configs-gen}/bin/rbe_configs_gen \ - --toolchain_container=rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} \ - --exec_os=linux \ - --target_os=linux \ - --bazel_version=${bazel_7.version} \ - --output_src_root=''${SRC_ROOT} \ - --output_config_path=generated-cc \ - --generate_java_configs=false \ - --generate_cpp_configs=true \ - --bazel_path=${bazel_7}/bin/bazel \ - --cpp_env_json=cpp_env.json - - # The rbe_configs_gen tool automatically sets the exec_properties of the - # generated platform to the generator container name and tag. For efficiency - # reasons the actual deployment won't be the same as this generator - # container, so we modify this in the generated configuration. - sed -i \ - 's|rbe-autogen-lre-cc|lre-cc|g' \ - ''${SRC_ROOT}/generated-cc/config/BUILD - - chmod 644 \ - ''${SRC_ROOT}/generated-cc/LICENSE \ - ''${SRC_ROOT}/generated-cc/config/BUILD \ - LRE_JAVA_IMAGE_TAG=$(nix eval .#lre-java.imageTag --raw) nix run .#rbe-autogen-lre-java.copyTo \ diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index 77620b080..f47277ac4 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -48,7 +48,7 @@ in { ++ [ # Integration testfiles not intended for production. "deployment-examples/docker-compose/example-do-not-use-in-prod-key.pem" - "kubernetes/components/insecure-certs/example-do-not-use-in-prod-key.pem" + "kubernetes/resources/insecure-certs/example-do-not-use-in-prod-key.pem" ]; enable = true; name = "detect-private-key"; diff --git a/tools/rules_rust-musl-platforms.diff b/tools/rules_rust-musl-platforms.diff new file mode 100644 index 000000000..e7429170c --- /dev/null +++ b/tools/rules_rust-musl-platforms.diff @@ -0,0 +1,13 @@ +diff --git a/rust/platform/triple_mappings.bzl b/rust/platform/triple_mappings.bzl +index b6191056..1261b529 100644 +--- a/rust/platform/triple_mappings.bzl ++++ b/rust/platform/triple_mappings.bzl +@@ -66,6 +66,8 @@ SUPPORTED_T2_PLATFORM_TRIPLES = { + "x86_64-unknown-freebsd": _support(std = True, host_tools = True), + "x86_64-unknown-fuchsia": _support(std = True, host_tools = False), + "x86_64-unknown-none": _support(std = True, host_tools = False), ++ "x86_64-unknown-linux-musl": _support(std = True, host_tools = False), ++ "aarch64-unknown-linux-musl": _support(std = True, host_tools = False), + } + + _T3_PLATFORM_TRIPLES = { diff --git a/web/platform/package.json b/web/platform/package.json index 2e348368e..fdaec4cee 100644 --- a/web/platform/package.json +++ b/web/platform/package.json @@ -10,7 +10,7 @@ "build": "bun fix && astro build", "docs": "bun run build.docs && bun run generate.docs", "generate.docs": "bun run utils/md_to_mdx_aot.ts", - "build.docs": "cd ../.. && unset TMPDIR TMP; bazel build nativelink-config:docs_json && cd web/platform && bun run utils/metaphase_aot.ts", + "build.docs": "cd ../.. && unset TMPDIR TMP; bazel build --@local-remote-execution//lre-rs:channel=nightly nativelink-config:docs_json && cd web/platform && bun run utils/metaphase_aot.ts", "check": "biome ci . && astro check", "check.format": "biome format .", "check.lint": "biome check .", diff --git a/web/platform/src/content/docs/docs/contribute/bazel.mdx b/web/platform/src/content/docs/docs/contribute/bazel.mdx index 6b7782536..109da2362 100644 --- a/web/platform/src/content/docs/docs/contribute/bazel.mdx +++ b/web/platform/src/content/docs/docs/contribute/bazel.mdx @@ -10,7 +10,10 @@ If you're using the Nix flake you're all set. If you're running outside of nix, install [bazelisk](https://github.com/bazelbuild/bazelisk/tree/master) manually and make sure you have a recent functional C++ toolchain with LLD as -linker. +linker. On Unix you'll also have to add +`--extra_toolchains=@rust_toolchains//:all` to all of your Bazel invocations to +allow the use of non-hermetic Rust toolchains. Windows is non-hermetic by +default. ## Build @@ -26,6 +29,12 @@ import { Tabs, TabItem } from '@astrojs/starlight/components'; $(pwd)/nativelink-config/examples/basic_cas.json ``` + + ```sh + bazel run --extra_toolchains=@rust_toolchains//:all nativelink -- \ + $(pwd)/nativelink-config/examples/basic_cas.json + ``` + ```sh bazel run --config=windows nativelink -- \ @@ -43,6 +52,12 @@ For optimized builds: $(pwd)/nativelink-config/examples/basic_cas.json ``` + + ```sh + bazel run -c opt --extra_toolchains=@rust_toolchains//:all nativelink -- \ + $(pwd)/nativelink-config/examples/basic_cas.json + ``` + ```sh bazel run --config=windows -c opt nativelink -- \ @@ -75,6 +90,11 @@ To run tests with Bazel: bazel test //... --verbose_failures ``` + + ```sh + bazel test --extra_toolchains=@rust_toolchains//:all //... --verbose_failures + ``` + ```sh bazel test --config=windows //... --verbose_failures