Implement Local Remote Execution for Rust

This change implements ~40 Rust toolchain configurations that reuse Nix's executables and make them available via Bazel. The new setup is fully reproducible, supports musl and glibc and works from all nix-supported hosts to all sensible crosscompilation targets seamlessly through remote and local execution. Until we have the actual infrastructure set up to fully test all configurations we have these 12 which are practically usable immediately: - MacOS native, aarch64 and x86_64, stable and nightly - Linux native, aarch64 and x64_64, stable and nightly, glibc and musl The channel is configurable via a new `--@local-remote-execution//rust:channel` flag while the libc version is configurable via the target platform. For instance: ```bash bazel build \ nativelink \ --@local-remote-execution//rust:channel=nightly \ --platforms=@local-remote-execution//rust/platforms:x86_64-unknown-linux-gnu bazel build \ nativelink \ --platforms=@local-remote-execution//rust/platforms:x86_64-unknown-linux-musl ```
TraceMachina · Dec 11, 2024 · 2a92b5a · 2a92b5a
1 parent a0ca341
commit 2a92b5a
Show file tree

Hide file tree

Showing 38 changed files with 1,863 additions and 156 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -13,3 +13,4 @@ bazel-testlogs
 bazel-nativelink
 local-remote-execution/generated-cc
 local-remote-execution/generated-java
+local-remote-execution/rust
diff --git a/.bazelrc b/.bazelrc
@@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh
 startup --windows_enable_symlinks
 build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
 build:windows --enable_runfiles
-
-# Global rust toolchain configuration. Deferred to here so that the cc
-# toolchains are resolved before the rust toolchains.
-build --extra_toolchains=@rust_toolchains//:all
+build:windows --extra_toolchains=@rust_toolchains//:all  # Doesn't support LRE.
 
 # Generated by the LRE flake module.
 try-import %workspace%/lre.bazelrc

diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -20,8 +20,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
-    name: Local / ${{ matrix.os }}
+        os: [ubuntu-24.04, macos-14]
+        toolchain: [lre-cc, lre-rs]
+        exclude:
+          - os: macos-14
+            toolchain: lre-cc
+    name: Local / ${{ matrix.toolchain }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     steps:
@@ -46,19 +50,22 @@ jobs:
         uses: >- # v4
           DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41
 
-      - name: Build hello_lre with LRE toolchain.
+      - name: Build example with ${{ matrix.toolchain }} toolchain
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "bazel run \
            --verbose_failures \
-           @local-remote-execution//examples:hello_lre"
+           @local-remote-execution//examples:${TOOLCHAIN}"
 
   remote:
     strategy:
       fail-fast: false
       matrix:
         os: [large-ubuntu-22.04]
-    name: Remote / ${{ matrix.os }}
+        toolchain: [lre-cc, lre-rs]
+    name: Remote / ${{ matrix.toolchain }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     steps:
@@ -92,12 +99,17 @@ jobs:
           REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || format('https://github.com/{0}.git', github.repository) }}
           BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
           COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: |
           nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF
           apiVersion: kustomize.config.k8s.io/v1beta1
           kind: Kustomization
           resources:
-          - deploy/kubernetes-example
+          - kubernetes/resources/flux
+          - kubernetes/resources/nativelink-core
+          - kubernetes/resources/gateway-routes
+          - kubernetes/workers/resources/worker-init
+          - kubernetes/workers/resources/${TOOLCHAIN}
           patches:
           - patch: |-
               - op: replace
@@ -112,6 +124,27 @@ jobs:
             target:
               kind: GitRepository
               name: nativelink
+          - patch: |-
+              - op: replace
+                path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+                value: ./src_root#image
+            target:
+              kind: Kustomization
+              name: nativelink-alert-core
+          - patch: |-
+              - op: replace
+                path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+                value: ./src_root#nativelink-worker-init
+            target:
+              kind: Kustomization
+              name: nativelink-alert-worker-init
+          - patch: |-
+              - op: replace
+                path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+                value: ./src_root#nativelink-worker-${TOOLCHAIN}
+            target:
+              kind: Kustomization
+              name: nativelink-alert-${TOOLCHAIN}
           EOF
           kubectl apply -k . &&
           rm kustomization.yaml'
@@ -124,6 +157,8 @@ jobs:
             nativelink-tekton-resources"
 
       - name: Wait for alerts
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
@@ -134,7 +169,7 @@ jobs:
             nativelink-alert-worker-init && \
           flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink-alert-lre-cc"
+            nativelink-alert-${TOOLCHAIN}"
 
       - name: Trigger pipelines
         run: |
@@ -181,21 +216,25 @@ jobs:
             nativelink-core"
 
       - name: Wait for Worker Kustomization
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "flux reconcile kustomization -n default \
             --timeout=15m \
-            nativelink-lre-cc"
+            nativelink-${TOOLCHAIN}"
 
       - name: Wait for NativeLink
         run: >
           nix develop --impure --command
           bash -c "kubectl rollout status deploy/nativelink"
 
       - name: Wait for worker
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
-          bash -c "kubectl rollout status deploy/nativelink-worker-lre-cc"
+          bash -c "kubectl rollout status deploy/nativelink-worker-${TOOLCHAIN}"
 
       - name: Get gateway IPs
         id: gateway-ips
@@ -214,11 +253,13 @@ jobs:
           echo "worker"
           kubectl logs -l app=nativelink-worker
 
-      - name: Build hello_lre with LRE toolchain.
+      - name: Build example with ${{ matrix.toolchain }} toolchain
+        env:
+          TOOLCHAIN: ${{ matrix.toolchain }}
         run: >
           nix develop --impure --command
           bash -c "bazel run \
             --remote_cache=grpc://$nativelink_ip \
             --remote_executor=grpc://$nativelink_ip \
             --verbose_failures \
-            @local-remote-execution//examples:hello_lre"
+            @local-remote-execution//examples:${TOOLCHAIN}"
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -159,13 +159,15 @@ jobs:
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
@@ -195,6 +197,7 @@ jobs:
         build-args: |
           OPT_LEVEL=fastbuild
           OS_VERSION=${{ matrix.os_version }}
+          ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
         load: true # This brings the build into `docker images` from buildx.
         tags: trace_machina/nativelink:latest
 

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -72,7 +72,9 @@ jobs:
       - name: Run Bazel tests
         run: |
           if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
-            bazel test //... --verbose_failures
+            bazel test //... \
+              --extra_toolchains=@rust_toolchains//:all \
+              --verbose_failures
           elif [ "$RUNNER_OS" == "Windows" ]; then
             bazel \
               --output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -6,7 +6,7 @@ module(
 
 bazel_dep(name = "rules_cc", version = "0.0.17")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_python", version = "0.36.0")
+bazel_dep(name = "rules_python", version = "0.40.0")
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
@@ -19,10 +19,50 @@ python.toolchain(
 use_repo(python, python = "python_versions")
 
 bazel_dep(name = "rules_rust", version = "0.54.1")
+archive_override(
+    module_name = "rules_rust",
+    integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=",
+    patch_strip = 1,
+    patches = ["//tools:rules_rust-musl-platforms.diff"],
+    urls = [
+        "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz",
+    ],
+)
+
+crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
+crate.from_cargo(
+    name = "crates",
+    cargo_lockfile = "//:Cargo.lock",
+    manifests = ["//:Cargo.toml"],
+    supported_platform_triples = [
+        "aarch64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "armv7-unknown-linux-gnueabi",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-msvc",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+    ],
+)
+use_repo(crate, "crates")
 
+# NativeLink uses Local Remote Execution for Rust by default which automatically
+# handles Rust toolchain configuration via Nix.
+#
+# If you build outside of Nix you'll have to register these toolchains
+# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your
+# Bazel invocation.
+#
+# WARNING: This configuration exists entirely as a convenience option and
+#          migration and is not a supported way of building production
+#          grade nativelink executables. It may be removed at any point in time.
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
 rust.toolchain(
     edition = "2021",
+
+    # These should always follow the versions from LRE.
     rust_analyzer_version = "nightly/2024-11-23",
     rustfmt_version = "nightly/2024-11-23",
     sha256s = {
@@ -44,43 +84,10 @@ rust.toolchain(
         "nightly/2024-11-23",
     ],
 )
-
-rust_host_tools = use_extension(
-    "@rules_rust//rust:extension.bzl",
-    "rust_host_tools",
-)
-rust_host_tools.host_tools(
-    edition = "2021",
-    version = "1.82.0",
-)
-
 use_repo(rust, "rust_toolchains")
 
-crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
-crate.from_cargo(
-    name = "crates",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = ["//:Cargo.toml"],
-    supported_platform_triples = [
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "arm-unknown-linux-gnueabi",
-        "armv7-unknown-linux-gnueabi",
-        "x86_64-apple-darwin",
-        "x86_64-pc-windows-msvc",
-        "x86_64-unknown-linux-gnu",
-    ],
-)
-use_repo(crate, "crates")
-
-rust_analyzer = use_extension(
-    "@rules_rust//tools/rust_analyzer:extension.bzl",
-    "rust_analyzer_dependencies",
-)
-rust_analyzer.rust_analyzer_dependencies()
-
-bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf")
-bazel_dep(name = "toolchains_protoc", version = "0.3.3")
+bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
+bazel_dep(name = "toolchains_protoc", version = "0.3.4")
 
 protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
 protoc.toolchain(

diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml
@@ -2,7 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../kubernetes-example
+- ../../kubernetes/resources/flux
+- ../../kubernetes/resources/nativelink-core
+- ../../kubernetes/resources/gateway-routes
+- ../../kubernetes/workers/resources/worker-init
+- ../../kubernetes/workers/resources/lre-cc
+- ../../kubernetes/workers/resources/lre-rs
 
 # Modify this value to change the repository containing the deployment files.
 #
@@ -52,3 +57,10 @@ patches:
   target:
     kind: Kustomization
     name: nativelink-alert-lre-cc
+- patch: |-
+    - op: replace
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
+      value: ./src_root#nativelink-worker-lre-rs
+  target:
+    kind: Kustomization
+    name: nativelink-alert-lre-rs