Implement Local Remote Execution for Rust

This change implements ~40 Rust toolchains that reuse Nix's executables and make them available via Bazel. The new setup is fully reproducible, supports musl and glibc and works from all nix-supported hosts to all sensible crosscompilation targets seamlessly through remote and local execution. For instance, you can now run this command on a MacOS host to build a statically linked executable for x86_64-linux on a remote arm worker: ```bash bazel build \ nativelink \ --extra_execution_platforms=@local-remote-execution//lre-rs:aarch64-unknown-linux-gnu \ --platforms=@local-remote-execution//lre-rs:x86_64-unknown-linux-musl ```
TraceMachina · Dec 4, 2024 · 3a3575a · 3a3575a
1 parent ae71bc8
commit 3a3575a
Show file tree

Hide file tree

Showing 24 changed files with 1,518 additions and 81 deletions.
diff --git a/.bazelignore b/.bazelignore
@@ -13,3 +13,4 @@ bazel-testlogs
 bazel-nativelink
 local-remote-execution/generated-cc
 local-remote-execution/generated-java
+local-remote-execution/lre-rs
diff --git a/.bazelrc b/.bazelrc
@@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh
 startup --windows_enable_symlinks
 build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
 build:windows --enable_runfiles
-
-# Global rust toolchain configuration. Deferred to here so that the cc
-# toolchains are resolved before the rust toolchains.
-build --extra_toolchains=@rust_toolchains//:all
+build:windows --extra_toolchains=@rust_toolchains//:all  # Doesn't support LRE.
 
 # Generated by the LRE flake module.
 try-import %workspace%/lre.bazelrc

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -159,13 +159,15 @@ jobs:
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
           bazel test //... \
+          --extra_toolchains=@rust_toolchains//:all \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
@@ -195,6 +197,7 @@ jobs:
         build-args: |
           OPT_LEVEL=fastbuild
           OS_VERSION=${{ matrix.os_version }}
+          ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
         load: true # This brings the build into `docker images` from buildx.
         tags: trace_machina/nativelink:latest
 

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -72,7 +72,9 @@ jobs:
       - name: Run Bazel tests
         run: |
           if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
-            bazel test //... --verbose_failures
+            bazel test //... \
+              --extra_toolchains=@rust_toolchains//:all \
+              --verbose_failures
           elif [ "$RUNNER_OS" == "Windows" ]; then
             bazel \
               --output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -6,7 +6,7 @@ module(
 
 bazel_dep(name = "rules_cc", version = "0.0.17")
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_python", version = "0.36.0")
+bazel_dep(name = "rules_python", version = "0.40.0")
 
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
@@ -19,10 +19,50 @@ python.toolchain(
 use_repo(python, python = "python_versions")
 
 bazel_dep(name = "rules_rust", version = "0.54.1")
+archive_override(
+    module_name = "rules_rust",
+    integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=",
+    patch_strip = 1,
+    patches = ["//tools:rules_rust-musl-platforms.diff"],
+    urls = [
+        "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz",
+    ],
+)
+
+crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
+crate.from_cargo(
+    name = "crates",
+    cargo_lockfile = "//:Cargo.lock",
+    manifests = ["//:Cargo.toml"],
+    supported_platform_triples = [
+        "aarch64-apple-darwin",
+        "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "armv7-unknown-linux-gnueabi",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-msvc",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
+    ],
+)
+use_repo(crate, "crates")
 
+# NativeLink uses Local Remote Execution for Rust by default which automatically
+# handles Rust toolchain configuration via Nix.
+#
+# If you build outside of Nix you'll have to register these toolchains
+# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your
+# Bazel invocation.
+#
+# WARNING: This configuration exists entirely as a convenience option and
+#          migration and is not a supported way of building production
+#          grade nativelink executables. It may be removed at any point in time.
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
 rust.toolchain(
     edition = "2021",
+
+    # These should always follow the versions from LRE.
     rust_analyzer_version = "nightly/2024-11-23",
     rustfmt_version = "nightly/2024-11-23",
     sha256s = {
@@ -44,43 +84,10 @@ rust.toolchain(
         "nightly/2024-11-23",
     ],
 )
-
-rust_host_tools = use_extension(
-    "@rules_rust//rust:extension.bzl",
-    "rust_host_tools",
-)
-rust_host_tools.host_tools(
-    edition = "2021",
-    version = "1.82.0",
-)
-
 use_repo(rust, "rust_toolchains")
 
-crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
-crate.from_cargo(
-    name = "crates",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = ["//:Cargo.toml"],
-    supported_platform_triples = [
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-gnu",
-        "arm-unknown-linux-gnueabi",
-        "armv7-unknown-linux-gnueabi",
-        "x86_64-apple-darwin",
-        "x86_64-pc-windows-msvc",
-        "x86_64-unknown-linux-gnu",
-    ],
-)
-use_repo(crate, "crates")
-
-rust_analyzer = use_extension(
-    "@rules_rust//tools/rust_analyzer:extension.bzl",
-    "rust_analyzer_dependencies",
-)
-rust_analyzer.rust_analyzer_dependencies()
-
-bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf")
-bazel_dep(name = "toolchains_protoc", version = "0.3.3")
+bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
+bazel_dep(name = "toolchains_protoc", version = "0.3.4")
 
 protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
 protoc.toolchain(

diff --git a/flake.nix b/flake.nix
@@ -52,20 +52,53 @@
         system,
         ...
       }: let
+        # C and C++ toolchain configuration. The basis of all toolchains.
+        llvmPackages = pkgs.llvmPackages_19;
+
+        # Rust toolchain configuration.
         stable-rust-version = "1.82.0";
         nightly-rust-version = "2024-11-23";
 
-        llvmPackages = pkgs.llvmPackages_19;
+        # This map translates execution platforms to sensible targets that can
+        # be built on such a platform. For instance, an x86_64-linux execution
+        # platform can target aarch64-linux and musl targets, but not darwin.
+        #
+        # On the Bazel side each key maps to a dedicated toolchain capture the
+        # cross-compile capabilities on the different exec platforms.
+        nixSystemToRustTargets = {
+          "aarch64-darwin" = [
+            "aarch64-apple-darwin"
+            "aarch64-unknown-linux-gnu"
+            "aarch64-unknown-linux-musl"
+            "x86_64-apple-darwin"
+            "x86_64-unknown-linux-gnu"
+            "x86_64-unknown-linux-musl"
+          ];
+          "aarch64-linux" = [
+            "aarch64-unknown-linux-gnu"
+            "aarch64-unknown-linux-musl"
+            "x86_64-unknown-linux-gnu"
+            "x86_64-unknown-linux-musl"
+          ];
+          "x86_64-darwin" = [
+            "aarch64-apple-darwin"
+            "aarch64-unknown-linux-gnu"
+            "aarch64-unknown-linux-musl"
+            "x86_64-apple-darwin"
+            "x86_64-unknown-linux-gnu"
+            "x86_64-unknown-linux-musl"
+          ];
+          "x86_64-linux" = [
+            "aarch64-unknown-linux-gnu"
+            "aarch64-unknown-linux-musl"
+            "x86_64-unknown-linux-gnu"
+            "x86_64-unknown-linux-musl"
+          ];
+        };
 
-        nixSystemToRustTriple = nixSystem:
-          {
-            "x86_64-linux" = "x86_64-unknown-linux-musl";
-            "aarch64-linux" = "aarch64-unknown-linux-musl";
-            "x86_64-darwin" = "x86_64-apple-darwin";
-            "aarch64-darwin" = "aarch64-apple-darwin";
-          }
-          .${nixSystem}
-          or (throw "Unsupported Nix system: ${nixSystem}");
+        nixExecToRustTargets = nixSystem:
+          nixSystemToRustTargets.${nixSystem}
+          or (throw "Unsupported Nix exec platform: ${nixSystem}");
 
         # Calling `pkgs.pkgsCross` changes the host and target platform to the
         # cross-target but leaves the build platform the same as pkgs.
@@ -80,24 +113,33 @@
         #
         # For optimal cache reuse of different crosscompilation toolchains we
         # take our rust toolchain from the host's `pkgs` and remap the rust
-        # target to the target platform of the `pkgsCross` target. This lets us
-        # reuse the same executables (for instance rustc) to build artifacts for
-        # different target platforms.
+        # target to sensible `pkgsCross` targets. This lets us reuse the same
+        # executables (for instance rustc) to build artifacts for different
+        # target platforms.
         stableRustFor = p:
           p.rust-bin.stable.${stable-rust-version}.default.override {
-            targets = [
-              "${nixSystemToRustTriple p.stdenv.targetPlatform.system}"
-            ];
+            targets = nixExecToRustTargets p.stdenv.targetPlatform.system;
           };
 
         nightlyRustFor = p:
           p.rust-bin.nightly.${nightly-rust-version}.default.override {
             extensions = ["llvm-tools"];
-            targets = [
-              "${nixSystemToRustTriple p.stdenv.targetPlatform.system}"
-            ];
+            targets = nixExecToRustTargets p.stdenv.targetPlatform.system;
           };
 
+        # This convenience method maps a nix system to the Rust target triple
+        # that we'd want to target by default. This is only used for Nix builds
+        # and doesn't have a Bazel equivalent.
+        nixSystemToRustTriple = nixSystem:
+          {
+            "x86_64-linux" = "x86_64-unknown-linux-musl";
+            "aarch64-linux" = "aarch64-unknown-linux-musl";
+            "x86_64-darwin" = "x86_64-apple-darwin";
+            "aarch64-darwin" = "aarch64-apple-darwin";
+          }
+          .${nixSystem}
+          or (throw "Unsupported Nix host platform: ${nixSystem}");
+
         craneLibFor = p: (crane.mkLib p).overrideToolchain stableRustFor;
         nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain nightlyRustFor;
 
@@ -248,6 +290,9 @@
           inherit (pkgs.lre) stdenv;
         };
         createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;};
+        gen-lre-rs = pkgs.callPackage ./tools/lre-rs.nix {
+          inherit nixSystemToRustTargets;
+        };
         buck2-toolchain = let
           buck2-nightly-rust-version = "2024-04-28";
           buck2-nightly-rust = pkgs.rust-bin.nightly.${buck2-nightly-rust-version};
@@ -288,6 +333,14 @@
         lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix {
           inherit buildImage;
         };
+        lre-rs = pkgs.callPackage ./local-remote-execution/lre-rs.nix {
+          inherit buildImage lre-cc;
+          rust-toolchains = [
+            (stableRustFor pkgs)
+            (nightlyRustFor pkgs)
+          ];
+        };
+
         toolchain-drake = buildImage {
           name = "toolchain-drake";
           # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds.
@@ -365,6 +418,7 @@
             inherit
               local-image-test
               lre-cc
+              lre-rs
               native-cli
               nativelink
               nativelinkCoverageForHost
@@ -376,13 +430,18 @@
               nativelink-x86_64-linux
               publish-ghcr
               ;
+
+            stable-rust = stableRustFor pkgs;
+            nightly-rust = nightlyRustFor pkgs;
+
             default = nativelink;
 
             rbe-autogen-lre-cc = rbe-autogen lre-cc;
             nativelink-worker-lre-cc = createWorker lre-cc;
             lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;};
             rbe-autogen-lre-java = rbe-autogen lre-java;
             nativelink-worker-lre-java = createWorker lre-java;
+            nativelink-worker-lre-rs = createWorker lre-rs;
             nativelink-worker-siso-chromium = createWorker siso-chromium;
             nativelink-worker-toolchain-drake = createWorker toolchain-drake;
             nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2;
@@ -422,9 +481,12 @@
         local-remote-execution.settings = {
           Env =
             if pkgs.stdenv.isDarwin
-            then [] # Doesn't support Darwin yet.
-            else lre-cc.meta.Env;
-          prefix = "linux";
+            then lre-rs.meta.Env # C++ doesn't support Darwin yet.
+            else (lre-cc.meta.Env ++ lre-rs.meta.Env);
+          prefix =
+            if pkgs.stdenv.isDarwin
+            then "macos"
+            else "linux";
         };
         nixos.settings = {
           path = with pkgs; [
@@ -451,6 +513,7 @@
               # Rust
               (stableRustFor pkgs)
               bazel
+              gen-lre-rs
 
               ## Infrastructure
               pkgs.awscli2

diff --git a/local-remote-execution/MODULE.bazel b/local-remote-execution/MODULE.bazel
@@ -14,3 +14,18 @@ bazel_dep(name = "rules_cc", version = "0.0.17")
 
 # Use the starlark implementation of Java rules instead of the builtin ones.
 bazel_dep(name = "rules_java", version = "8.5.1")
+bazel_dep(name = "rules_rust", version = "0.54.1")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
+
+lre_rs = use_extension("//lre-rs:extension.bzl", "lre_rs")
+use_repo(
+    lre_rs,
+    "lre-rs-nightly-aarch64-darwin",
+    "lre-rs-nightly-aarch64-linux",
+    "lre-rs-nightly-x86_64-darwin",
+    "lre-rs-nightly-x86_64-linux",
+    "lre-rs-stable-aarch64-darwin",
+    "lre-rs-stable-aarch64-linux",
+    "lre-rs-stable-x86_64-darwin",
+    "lre-rs-stable-x86_64-linux",
+)
diff --git a/local-remote-execution/libc/BUILD.bazel b/local-remote-execution/libc/BUILD.bazel
@@ -0,0 +1,13 @@
+constraint_setting(name = "libc")
+
+constraint_value(
+    name = "glibc",
+    constraint_setting = ":libc",
+    visibility = ["//visibility:public"],
+)
+
+constraint_value(
+    name = "musl",
+    constraint_setting = ":libc",
+    visibility = ["//visibility:public"],
+)