Implement Local Remote Execution for Rust

This change implements ~40 Rust toolchain configurations that reuse Nix's executables and make them available via Bazel. The new setup is fully reproducible, supports musl and glibc and works from all nix-supported hosts to all sensible crosscompilation targets seamlessly through remote and local execution. Until we have the actual infrastructure set up to fully test all configurations we have these 12 which are practically usable immediately: - MacOS native, aarch64 and x86_64, stable and nightly - Linux native, aarch64 and x64_64, stable and nightly, glibc and musl The channel is configurable via a new `--@local-remote-execution//lre-rs:channel` flag while the libc version is configurable via the target platform. For instance: ```bash bazel build \ nativelink \ --@local-remote-execution//lre-rs:channel=nightly \ --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-gnu bazel build \ nativelink \ --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-musl ```
TraceMachina · Dec 9, 2024 · 408148d · 408148d
1 parent c84f793
commit 408148d
Show file tree

Hide file tree

Showing 85 changed files with 2,360 additions and 456 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -13,3 +13,4 @@ bazel-testlogs
 bazel-nativelink
 local-remote-execution/generated-cc
 local-remote-execution/generated-java
+local-remote-execution/lre-rs
diff --git a/.bazelrc b/.bazelrc
@@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh
 startup --windows_enable_symlinks
 build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
 build:windows --enable_runfiles
-
-# Global rust toolchain configuration. Deferred to here so that the cc
-# toolchains are resolved before the rust toolchains.
-build --extra_toolchains=@rust_toolchains//:all
+build:windows --extra_toolchains=@rust_toolchains//:all  # Doesn't support LRE.
 
 # Generated by the LRE flake module.
 try-import %workspace%/lre.bazelrc

diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -96,16 +96,9 @@ jobs:
           nix develop --impure --command bash -c 'cat > kustomization.yaml << EOF
           apiVersion: kustomize.config.k8s.io/v1beta1
           kind: Kustomization
-          components:
-          - kubernetes/components/operator
+          resources:
+          - kubernetes/overlays/lre-flux
           patches:
-          - patch: |-
-              - op: replace
-                path: /spec/path
-                value: ./kubernetes/overlays/lre
-            target:
-              kind: Kustomization
-              name: nativelink
           - patch: |-
               - op: replace
                 path: /spec/url
@@ -119,27 +112,6 @@ jobs:
             target:
               kind: GitRepository
               name: nativelink
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#image
-            target:
-              kind: Alert
-              name: nativelink-image-alert
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#nativelink-worker-init
-            target:
-              kind: Alert
-              name: nativelink-worker-init-alert
-          - patch: |-
-              - op: replace
-                path: /spec/eventMetadata/flakeOutput
-                value: ./src_root#nativelink-worker-lre-cc
-            target:
-              kind: Alert
-              name: nativelink-worker-alert
           EOF
           kubectl apply -k . &&
           rm kustomization.yaml'
@@ -151,6 +123,36 @@ jobs:
             --timeout=15m \
             nativelink-tekton-resources"
 
+      - name: Wait for alerts
+        run: >
+          nix develop --impure --command
+          bash -c "flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-core && \
+          flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-worker-init && \
+          flux reconcile kustomization -n default \
+            --timeout=15m \
+            nativelink-alert-lre-cc"
+
+      - name: Trigger pipelines
+        run: |
+          nix develop --impure --command bash -c 'cat > dummy-repo.yaml << EOF
+          apiVersion: source.toolkit.fluxcd.io/v1
+          kind: GitRepository
+          metadata:
+            name: dummy-repository
+            namespace: default
+          spec:
+            interval: 2m
+            url: https://github.com/TraceMachina/nativelink
+            ref:
+              branch: main
+          EOF
+          kubectl apply -f dummy-repo.yaml &&
+          rm dummy-repo.yaml'
+
       - name: Wait for Tekton pipelines
         run: >
           nix develop --impure --command bash << 'EOF'
@@ -171,13 +173,6 @@ jobs:
               -l tekton.dev/pipeline=rebuild-nativelink
           EOF
 
-      - name: Wait for Configmaps
-        run: >
-          nix develop --impure --command
-          bash -c "flux reconcile kustomization -n default \
-            --timeout=15m \
-            nativelink-configmaps"
-
       - name: Wait for NativeLink Kustomization
         run: >
           nix develop --impure --command

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -159,13 +159,15 @@ jobs:
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
@@ -195,6 +197,7 @@ jobs:
         build-args: |
           OPT_LEVEL=fastbuild
           OS_VERSION=${{ matrix.os_version }}
+          ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
         load: true # This brings the build into `docker images` from buildx.
         tags: trace_machina/nativelink:latest
 

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -72,7 +72,9 @@ jobs:
       - name: Run Bazel tests
         run: |
           if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
-            bazel test //... --verbose_failures
+            bazel test //... \
+              --extra_toolchains=@rust_toolchains//:all \
+              --verbose_failures
           elif [ "$RUNNER_OS" == "Windows" ]; then
             bazel \
               --output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -6,7 +6,7 @@ module(
 
 bazel_dep(name = "rules_cc", version = "0.0.17")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_python", version = "0.36.0")
+bazel_dep(name = "rules_python", version = "0.40.0")
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
@@ -19,10 +19,50 @@ python.toolchain(
 use_repo(python, python = "python_versions")
 
 bazel_dep(name = "rules_rust", version = "0.54.1")
+archive_override(
+    module_name = "rules_rust",
+    integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=",
+    patch_strip = 1,
+    patches = ["//tools:rules_rust-musl-platforms.diff"],
+    urls = [
+        "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz",
+    ],
+)
+
+crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
+crate.from_cargo(
+    name = "crates",
+    cargo_lockfile = "//:Cargo.lock",
+    manifests = ["//:Cargo.toml"],
+    supported_platform_triples = [
+        "aarch64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "armv7-unknown-linux-gnueabi",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-msvc",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+    ],
+)
+use_repo(crate, "crates")
 
+# NativeLink uses Local Remote Execution for Rust by default which automatically
+# handles Rust toolchain configuration via Nix.
+#
+# If you build outside of Nix you'll have to register these toolchains
+# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your
+# Bazel invocation.
+#
+# WARNING: This configuration exists entirely as a convenience option and
+#          migration and is not a supported way of building production
+#          grade nativelink executables. It may be removed at any point in time.
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
 rust.toolchain(
     edition = "2021",
+
+    # These should always follow the versions from LRE.
     rust_analyzer_version = "nightly/2024-11-23",
     rustfmt_version = "nightly/2024-11-23",
     sha256s = {
@@ -44,43 +84,10 @@ rust.toolchain(
         "nightly/2024-11-23",
     ],
 )
-
-rust_host_tools = use_extension(
-    "@rules_rust//rust:extension.bzl",
-    "rust_host_tools",
-)
-rust_host_tools.host_tools(
-    edition = "2021",
-    version = "1.82.0",
-)
-
 use_repo(rust, "rust_toolchains")
 
-crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
-crate.from_cargo(
-    name = "crates",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = ["//:Cargo.toml"],
-    supported_platform_triples = [
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "arm-unknown-linux-gnueabi",
-        "armv7-unknown-linux-gnueabi",
-        "x86_64-apple-darwin",
-        "x86_64-pc-windows-msvc",
-        "x86_64-unknown-linux-gnu",
-    ],
-)
-use_repo(crate, "crates")
-
-rust_analyzer = use_extension(
-    "@rules_rust//tools/rust_analyzer:extension.bzl",
-    "rust_analyzer_dependencies",
-)
-rust_analyzer.rust_analyzer_dependencies()
-
-bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf")
-bazel_dep(name = "toolchains_protoc", version = "0.3.3")
+bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
+bazel_dep(name = "toolchains_protoc", version = "0.3.4")
 
 protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
 protoc.toolchain(

diff --git a/deploy/dev/kustomization.yaml b/deploy/dev/kustomization.yaml
@@ -1,33 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-components:
-- ../../kubernetes/components/operator
+resources:
+- ../../kubernetes/overlays/lre-flux
 
-# Change this value to deploy custom overlays.
-patches:
-- patch: |-
-    - op: replace
-      path: /spec/path
-      value: ./kubernetes/overlays/lre
-  target:
-    kind: Kustomization
-    name: nativelink
-
-# Modify this value to change the URL of the repository with deployment files.
+# Modify this value to change the repository containing the deployment files.
 #
 # This is usually only necessary if you change deployment YAML files or
 # NativeLink config files. If you only intend to change the Rust sources you can
 # leave this as is and need to ensure that the Alerts below are patched to build
 # your local sources.
+patches:
 - patch: |-
     - op: replace
       path: /spec/url
       value: https://github.com/TraceMachina/nativelink
-# Optionally, change the tracked branch.
-#   - op: replace
-#     path: /spec/ref/branch
-#     value: somecustombranch
+    - op: replace
+      path: /spec/ref/branch
+      value: main
   target:
     kind: GitRepository
     name: nativelink
@@ -43,22 +33,22 @@ patches:
 # outputs from a Pull request.
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#image
   target:
-    kind: Alert
-    name: nativelink-image-alert
+    kind: Kustomization
+    name: nativelink-alert-core
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#nativelink-worker-init
   target:
-    kind: Alert
-    name: nativelink-worker-init-alert
+    kind: Kustomization
+    name: nativelink-alert-worker-init
 - patch: |-
     - op: replace
-      path: /spec/eventMetadata/flakeOutput
+      path: /spec/postBuild/substitute/PLACEHOLDER_FLAKE_OUTPUT
       value: ./src_root#nativelink-worker-lre-cc
   target:
-    kind: Alert
-    name: nativelink-worker-alert
+    kind: Kustomization
+    name: nativelink-alert-lre-cc