Implement Local Remote Execution for Rust

This change implements ~40 Rust toolchain configurations that reuse Nix's executables and make them available via Bazel. The new setup is fully reproducible, supports musl and glibc and works from all nix-supported hosts to all sensible crosscompilation targets seamlessly through remote and local execution. Until we have the actual infrastructure set up to fully test all configurations we have these 12 which are practically usable immediately: - MacOS native, aarch64 and x86_64, stable and nightly - Linux native, aarch64 and x64_64, stable and nightly, glibc and musl The channel is configurable via a new `--@local-remote-execution//lre-rs:channel` flag while the libc version is configurable via the target platform. For instance: ```bash bazel build \ nativelink \ --@local-remote-execution//lre-rs:channel=nightly \ --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-gnu bazel build \ nativelink \ --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-musl ```
TraceMachina · Dec 8, 2024 · 1669a8e · 1669a8e
1 parent c84f793
commit 1669a8e
Show file tree

Hide file tree

Showing 82 changed files with 2,259 additions and 400 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -13,3 +13,4 @@ bazel-testlogs
 bazel-nativelink
 local-remote-execution/generated-cc
 local-remote-execution/generated-java
+local-remote-execution/lre-rs
diff --git a/.bazelrc b/.bazelrc
@@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh
 startup --windows_enable_symlinks
 build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
 build:windows --enable_runfiles
-
-# Global rust toolchain configuration. Deferred to here so that the cc
-# toolchains are resolved before the rust toolchains.
-build --extra_toolchains=@rust_toolchains//:all
+build:windows --extra_toolchains=@rust_toolchains//:all  # Doesn't support LRE.
 
 # Generated by the LRE flake module.
 try-import %workspace%/lre.bazelrc

diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -171,13 +171,6 @@ jobs:
               -l tekton.dev/pipeline=rebuild-nativelink
           EOF
 
-      - name: Wait for Configmaps
-        run: >
-          nix develop --impure --command
-          bash -c "flux reconcile kustomization -n default \
-            --timeout=15m \
-            nativelink-configmaps"
-
       - name: Wait for NativeLink Kustomization
         run: >
           nix develop --impure --command

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -159,13 +159,15 @@ jobs:
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
@@ -195,6 +197,7 @@ jobs:
         build-args: |
           OPT_LEVEL=fastbuild
           OS_VERSION=${{ matrix.os_version }}
+          ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
         load: true # This brings the build into `docker images` from buildx.
         tags: trace_machina/nativelink:latest
 

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -72,7 +72,9 @@ jobs:
       - name: Run Bazel tests
         run: |
           if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
-            bazel test //... --verbose_failures
+            bazel test //... \
+              --extra_toolchains=@rust_toolchains//:all \
+              --verbose_failures
           elif [ "$RUNNER_OS" == "Windows" ]; then
             bazel \
               --output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -6,7 +6,7 @@ module(
 
 bazel_dep(name = "rules_cc", version = "0.0.17")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_python", version = "0.36.0")
+bazel_dep(name = "rules_python", version = "0.40.0")
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
@@ -19,10 +19,50 @@ python.toolchain(
 use_repo(python, python = "python_versions")
 
 bazel_dep(name = "rules_rust", version = "0.54.1")
+archive_override(
+    module_name = "rules_rust",
+    integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=",
+    patch_strip = 1,
+    patches = ["//tools:rules_rust-musl-platforms.diff"],
+    urls = [
+        "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz",
+    ],
+)
+
+crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
+crate.from_cargo(
+    name = "crates",
+    cargo_lockfile = "//:Cargo.lock",
+    manifests = ["//:Cargo.toml"],
+    supported_platform_triples = [
+        "aarch64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "armv7-unknown-linux-gnueabi",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-msvc",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+    ],
+)
+use_repo(crate, "crates")
 
+# NativeLink uses Local Remote Execution for Rust by default which automatically
+# handles Rust toolchain configuration via Nix.
+#
+# If you build outside of Nix you'll have to register these toolchains
+# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your
+# Bazel invocation.
+#
+# WARNING: This configuration exists entirely as a convenience option and
+#          migration and is not a supported way of building production
+#          grade nativelink executables. It may be removed at any point in time.
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
 rust.toolchain(
     edition = "2021",
+
+    # These should always follow the versions from LRE.
     rust_analyzer_version = "nightly/2024-11-23",
     rustfmt_version = "nightly/2024-11-23",
     sha256s = {
@@ -44,43 +84,10 @@ rust.toolchain(
         "nightly/2024-11-23",
     ],
 )
-
-rust_host_tools = use_extension(
-    "@rules_rust//rust:extension.bzl",
-    "rust_host_tools",
-)
-rust_host_tools.host_tools(
-    edition = "2021",
-    version = "1.82.0",
-)
-
 use_repo(rust, "rust_toolchains")
 
-crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
-crate.from_cargo(
-    name = "crates",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = ["//:Cargo.toml"],
-    supported_platform_triples = [
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "arm-unknown-linux-gnueabi",
-        "armv7-unknown-linux-gnueabi",
-        "x86_64-apple-darwin",
-        "x86_64-pc-windows-msvc",
-        "x86_64-unknown-linux-gnu",
-    ],
-)
-use_repo(crate, "crates")
-
-rust_analyzer = use_extension(
-    "@rules_rust//tools/rust_analyzer:extension.bzl",
-    "rust_analyzer_dependencies",
-)
-rust_analyzer.rust_analyzer_dependencies()
-
-bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf")
-bazel_dep(name = "toolchains_protoc", version = "0.3.3")
+bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
+bazel_dep(name = "toolchains_protoc", version = "0.3.4")
 
 protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
 protoc.toolchain(

diff --git a/flake.nix b/flake.nix
@@ -52,54 +52,8 @@
         system,
         ...
       }: let
-        stable-rust-version = "1.82.0";
-        nightly-rust-version = "2024-11-23";
-
-        llvmPackages = pkgs.llvmPackages_19;
-
-        nixSystemToRustTriple = nixSystem:
-          {
-            "x86_64-linux" = "x86_64-unknown-linux-musl";
-            "aarch64-linux" = "aarch64-unknown-linux-musl";
-            "x86_64-darwin" = "x86_64-apple-darwin";
-            "aarch64-darwin" = "aarch64-apple-darwin";
-          }
-          .${nixSystem}
-          or (throw "Unsupported Nix system: ${nixSystem}");
-
-        # Calling `pkgs.pkgsCross` changes the host and target platform to the
-        # cross-target but leaves the build platform the same as pkgs.
-        #
-        # For instance, calling `pkgs.pkgsCross.aarch64-multiplatform` on an
-        # `x86_64-linux` host sets `host==target==aarch64-linux` but leaves the
-        # build platform at `x86_64-linux`.
-        #
-        # On aarch64-darwin the same `pkgs.pkgsCross.aarch64-multiplatform`
-        # again sets `host==target==aarch64-linux` but now with a build platform
-        # of `aarch64-darwin`.
-        #
-        # For optimal cache reuse of different crosscompilation toolchains we
-        # take our rust toolchain from the host's `pkgs` and remap the rust
-        # target to the target platform of the `pkgsCross` target. This lets us
-        # reuse the same executables (for instance rustc) to build artifacts for
-        # different target platforms.
-        stableRustFor = p:
-          p.rust-bin.stable.${stable-rust-version}.default.override {
-            targets = [
-              "${nixSystemToRustTriple p.stdenv.targetPlatform.system}"
-            ];
-          };
-
-        nightlyRustFor = p:
-          p.rust-bin.nightly.${nightly-rust-version}.default.override {
-            extensions = ["llvm-tools"];
-            targets = [
-              "${nixSystemToRustTriple p.stdenv.targetPlatform.system}"
-            ];
-          };
-
-        craneLibFor = p: (crane.mkLib p).overrideToolchain stableRustFor;
-        nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain nightlyRustFor;
+        craneLibFor = p: (crane.mkLib p).overrideToolchain pkgs.lre.stableRustFor;
+        nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain pkgs.lre.nightlyRustFor;
 
         src = pkgs.lib.cleanSourceWith {
           src = (craneLibFor pkgs).path ./.;
@@ -110,18 +64,32 @@
 
         # Warning: The different usages of `p` and `pkgs` are intentional as we
         # use crosscompilers and crosslinkers whose packagesets collapse with
-        # the host's packageset. If you change this, take care that you don't
+        # the host's packageset. If you change this take care that you don't
         # accidentally explode the global closure size.
         commonArgsFor = p: let
           isLinuxBuild = p.stdenv.buildPlatform.isLinux;
           isLinuxTarget = p.stdenv.targetPlatform.isLinux;
-          targetArch = nixSystemToRustTriple p.stdenv.targetPlatform.system;
+          # Map the nix system to the Rust target triple that we'd want to target
+          # by default.
+          targetArch =
+            (
+              nixSystem:
+                {
+                  "x86_64-linux" = "x86_64-unknown-linux-musl";
+                  "aarch64-linux" = "aarch64-unknown-linux-musl";
+                  "x86_64-darwin" = "x86_64-apple-darwin";
+                  "aarch64-darwin" = "aarch64-apple-darwin";
+                }
+                .${nixSystem}
+                or (throw "Unsupported Nix host platform: ${nixSystem}")
+            )
+            p.stdenv.targetPlatform.system;
 
           # Full path to the linker for CARGO_TARGET_XXX_LINKER
           linkerPath =
             if isLinuxBuild && isLinuxTarget
             then "${pkgs.mold}/bin/ld.mold"
-            else "${llvmPackages.lld}/bin/ld.lld";
+            else "${pkgs.llvmPackages_19.lld}/bin/ld.lld";
 
           linkerEnvVar = "CARGO_TARGET_${pkgs.lib.toUpper (pkgs.lib.replaceStrings ["-"] ["_"] targetArch)}_LINKER";
         in
@@ -142,7 +110,7 @@
               (
                 if isLinuxBuild
                 then [pkgs.mold]
-                else [llvmPackages.lld]
+                else [pkgs.llvmPackages_19.lld]
               )
               ++ pkgs.lib.optionals p.stdenv.targetPlatform.isDarwin [
                 p.darwin.apple_sdk.frameworks.Security
@@ -193,18 +161,14 @@
 
         nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;};
 
-        rbe-configs-gen = pkgs.callPackage ./local-remote-execution/rbe-configs-gen.nix {};
-
-        generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {inherit rbe-configs-gen;};
-
         native-cli = pkgs.callPackage ./native-cli/default.nix {};
 
         build-chromium-tests =
           pkgs.writeShellScriptBin
           "build-chromium-tests"
           ./deploy/chromium-example/build_chromium_tests.sh;
 
-        docs = pkgs.callPackage ./tools/docs.nix {rust = stableRustFor pkgs;};
+        docs = pkgs.callPackage ./tools/docs.nix {rust = pkgs.lre.stable-rust;};
 
         inherit (nix2container.packages.${system}.nix2container) pullImage;
         inherit (nix2container.packages.${system}.nix2container) buildImage;
@@ -243,10 +207,6 @@
 
         nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;};
 
-        rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix {
-          inherit buildImage;
-          inherit (pkgs.lre) stdenv;
-        };
         createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;};
         buck2-toolchain = let
           buck2-nightly-rust-version = "2024-04-28";
@@ -285,9 +245,7 @@
             os = "linux";
           };
         };
-        lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix {
-          inherit buildImage;
-        };
+
         toolchain-drake = buildImage {
           name = "toolchain-drake";
           # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds.
@@ -364,7 +322,6 @@
           rec {
             inherit
               local-image-test
-              lre-cc
               native-cli
               nativelink
               nativelinkCoverageForHost
@@ -376,13 +333,14 @@
               nativelink-x86_64-linux
               publish-ghcr
               ;
+
             default = nativelink;
 
-            rbe-autogen-lre-cc = rbe-autogen lre-cc;
-            nativelink-worker-lre-cc = createWorker lre-cc;
+            nativelink-worker-lre-cc = createWorker pkgs.lre.lre-cc.image;
             lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;};
-            rbe-autogen-lre-java = rbe-autogen lre-java;
+            rbe-autogen-lre-java = pkgs.rbe-autogen lre-java;
             nativelink-worker-lre-java = createWorker lre-java;
+            nativelink-worker-lre-rs = createWorker pkgs.lre.lre-rs.image;
             nativelink-worker-siso-chromium = createWorker siso-chromium;
             nativelink-worker-toolchain-drake = createWorker toolchain-drake;
             nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2;
@@ -416,15 +374,18 @@
         pre-commit.settings = {
           hooks = import ./tools/pre-commit-hooks.nix {
             inherit pkgs;
-            nightly-rust = pkgs.rust-bin.nightly.${nightly-rust-version};
+            nightly-rust = pkgs.rust-bin.nightly.${pkgs.lre.nightly-rust.meta.version};
           };
         };
         local-remote-execution.settings = {
-          Env =
+          Env = with pkgs.lre;
+            if pkgs.stdenv.isDarwin
+            then lre-rs.meta.Env # C++ doesn't support Darwin yet.
+            else (lre-cc.meta.Env ++ lre-rs.meta.Env);
+          prefix =
             if pkgs.stdenv.isDarwin
-            then [] # Doesn't support Darwin yet.
-            else lre-cc.meta.Env;
-          prefix = "linux";
+            then "macos"
+            else "linux";
         };
         nixos.settings = {
           path = with pkgs; [
@@ -449,8 +410,9 @@
               pkgs.pre-commit
 
               # Rust
-              (stableRustFor pkgs)
               bazel
+              pkgs.lre.stable-rust
+              pkgs.lre.lre-rs.lre-rs-configs-gen
 
               ## Infrastructure
               pkgs.awscli2
@@ -481,7 +443,7 @@
 
               # Additional tools from within our development environment.
               local-image-test
-              generate-toolchains
+              pkgs.lre.lre-cc.lre-cc-configs-gen
               pkgs.lre.clang
               native-cli
               docs
@@ -546,7 +508,7 @@
         nixos = ./tools/nixos/flake-module.nix;
       };
       overlays = {
-        lre = import ./local-remote-execution/overlays/default.nix;
+        lre = import ./local-remote-execution/overlays/default.nix {inherit nix2container;};
       };
     };
 }