Skip to content

Commit

Permalink
Implement Local Remote Execution for Rust
Browse files Browse the repository at this point in the history
This change implements ~40 Rust toolchain configurations that reuse
Nix's executables and make them available via Bazel.

The new setup is fully reproducible, supports musl and glibc and
works from all nix-supported hosts to all sensible crosscompilation
targets seamlessly through remote and local execution.

Until we have the actual infrastructure set up to fully test all
configurations we have these 12 which are practically usable
immediately:

- MacOS native, aarch64 and x86_64, stable and nightly
- Linux native, aarch64 and x64_64, stable and nightly, glibc and musl

The channel is configurable via a new
`--@local-remote-execution//lre-rs:channel` flag while the libc version
is configurable via the target platform. For instance:

```bash
bazel build \
    nativelink \
    --@local-remote-execution//lre-rs:channel=nightly \
    --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-gnu

bazel build \
    nativelink \
    --platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-musl

bazel build nativelink
```
  • Loading branch information
aaronmondal committed Dec 5, 2024
1 parent ae71bc8 commit e686526
Show file tree
Hide file tree
Showing 26 changed files with 1,573 additions and 112 deletions.
1 change: 1 addition & 0 deletions .bazelignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ bazel-testlogs
bazel-nativelink
local-remote-execution/generated-cc
local-remote-execution/generated-java
local-remote-execution/lre-rs
5 changes: 1 addition & 4 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,7 @@ build:tsan --run_under=//tools:tsan.sh
startup --windows_enable_symlinks
build:windows --cxxopt=/std:c++14 --host_cxxopt=/std:c++14
build:windows --enable_runfiles

# Global rust toolchain configuration. Deferred to here so that the cc
# toolchains are resolved before the rust toolchains.
build --extra_toolchains=@rust_toolchains//:all
build:windows --extra_toolchains=@rust_toolchains//:all # Doesn't support LRE.

# Generated by the LRE flake module.
try-import %workspace%/lre.bazelrc
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,15 @@ jobs:
docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
bazel clean && \
bazel test //... \
--extra_toolchains=@rust_toolchains//:all \
--remote_cache=grpc://127.0.0.1:50051 \
--remote_executor=grpc://127.0.0.1:50052 \
--remote_default_exec_properties=cpu_count=1 \
' && \
docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
bazel clean && \
bazel test //... \
--extra_toolchains=@rust_toolchains//:all \
--remote_cache=grpc://127.0.0.1:50051 \
--remote_executor=grpc://127.0.0.1:50052 \
--remote_default_exec_properties=cpu_count=1 \
Expand Down Expand Up @@ -195,6 +197,7 @@ jobs:
build-args: |
OPT_LEVEL=fastbuild
OS_VERSION=${{ matrix.os_version }}
ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:latest

Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/native-bazel.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ jobs:
- name: Run Bazel tests
run: |
if [ "$RUNNER_OS" == "Linux" ] || [ "$RUNNER_OS" == "macOS" ]; then
bazel test //... --verbose_failures
bazel test //... \
--extra_toolchains=@rust_toolchains//:all \
--verbose_failures
elif [ "$RUNNER_OS" == "Windows" ]; then
bazel \
--output_user_root=${{ steps.bazel-cache.outputs.mountpoint }} \
Expand Down
79 changes: 43 additions & 36 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ module(

bazel_dep(name = "rules_cc", version = "0.0.17")
bazel_dep(name = "platforms", version = "0.0.10")
bazel_dep(name = "rules_python", version = "0.36.0")
bazel_dep(name = "rules_python", version = "0.40.0")

python = use_extension("@rules_python//python/extensions:python.bzl", "python")
python.toolchain(
Expand All @@ -19,10 +19,50 @@ python.toolchain(
use_repo(python, python = "python_versions")

bazel_dep(name = "rules_rust", version = "0.54.1")
archive_override(
module_name = "rules_rust",
integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=",
patch_strip = 1,
patches = ["//tools:rules_rust-musl-platforms.diff"],
urls = [
"https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz",
],
)

crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
crate.from_cargo(
name = "crates",
cargo_lockfile = "//:Cargo.lock",
manifests = ["//:Cargo.toml"],
supported_platform_triples = [
"aarch64-apple-darwin",
"aarch64-unknown-linux-gnu",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-gnueabi",
"armv7-unknown-linux-gnueabi",
"x86_64-apple-darwin",
"x86_64-pc-windows-msvc",
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
],
)
use_repo(crate, "crates")

# NativeLink uses Local Remote Execution for Rust by default which automatically
# handles Rust toolchain configuration via Nix.
#
# If you build outside of Nix you'll have to register these toolchains
# explicitly by passing `--extra_toolchains=@rust_toolchains//:all` to your
# Bazel invocation.
#
# WARNING: This configuration exists entirely as a convenience option and
# migration and is not a supported way of building production
# grade nativelink executables. It may be removed at any point in time.
rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
rust.toolchain(
edition = "2021",

# These should always follow the versions from LRE.
rust_analyzer_version = "nightly/2024-11-23",
rustfmt_version = "nightly/2024-11-23",
sha256s = {
Expand All @@ -44,43 +84,10 @@ rust.toolchain(
"nightly/2024-11-23",
],
)

rust_host_tools = use_extension(
"@rules_rust//rust:extension.bzl",
"rust_host_tools",
)
rust_host_tools.host_tools(
edition = "2021",
version = "1.82.0",
)

use_repo(rust, "rust_toolchains")

crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate")
crate.from_cargo(
name = "crates",
cargo_lockfile = "//:Cargo.lock",
manifests = ["//:Cargo.toml"],
supported_platform_triples = [
"aarch64-apple-darwin",
"aarch64-unknown-linux-gnu",
"arm-unknown-linux-gnueabi",
"armv7-unknown-linux-gnueabi",
"x86_64-apple-darwin",
"x86_64-pc-windows-msvc",
"x86_64-unknown-linux-gnu",
],
)
use_repo(crate, "crates")

rust_analyzer = use_extension(
"@rules_rust//tools/rust_analyzer:extension.bzl",
"rust_analyzer_dependencies",
)
rust_analyzer.rust_analyzer_dependencies()

bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf")
bazel_dep(name = "toolchains_protoc", version = "0.3.3")
bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
bazel_dep(name = "toolchains_protoc", version = "0.3.4")

protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
protoc.toolchain(
Expand Down
73 changes: 30 additions & 43 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,12 @@
system,
...
}: let
stable-rust-version = "1.82.0";
nightly-rust-version = "2024-11-23";

# C and C++ toolchain configuration. The basis of all toolchains.
llvmPackages = pkgs.llvmPackages_19;

# This convenience method maps a nix system to the Rust target triple
# that we'd want to target by default. This is only used for Nix builds
# and doesn't have a Bazel equivalent.
nixSystemToRustTriple = nixSystem:
{
"x86_64-linux" = "x86_64-unknown-linux-musl";
Expand All @@ -65,41 +66,10 @@
"aarch64-darwin" = "aarch64-apple-darwin";
}
.${nixSystem}
or (throw "Unsupported Nix system: ${nixSystem}");

# Calling `pkgs.pkgsCross` changes the host and target platform to the
# cross-target but leaves the build platform the same as pkgs.
#
# For instance, calling `pkgs.pkgsCross.aarch64-multiplatform` on an
# `x86_64-linux` host sets `host==target==aarch64-linux` but leaves the
# build platform at `x86_64-linux`.
#
# On aarch64-darwin the same `pkgs.pkgsCross.aarch64-multiplatform`
# again sets `host==target==aarch64-linux` but now with a build platform
# of `aarch64-darwin`.
#
# For optimal cache reuse of different crosscompilation toolchains we
# take our rust toolchain from the host's `pkgs` and remap the rust
# target to the target platform of the `pkgsCross` target. This lets us
# reuse the same executables (for instance rustc) to build artifacts for
# different target platforms.
stableRustFor = p:
p.rust-bin.stable.${stable-rust-version}.default.override {
targets = [
"${nixSystemToRustTriple p.stdenv.targetPlatform.system}"
];
};

nightlyRustFor = p:
p.rust-bin.nightly.${nightly-rust-version}.default.override {
extensions = ["llvm-tools"];
targets = [
"${nixSystemToRustTriple p.stdenv.targetPlatform.system}"
];
};
or (throw "Unsupported Nix host platform: ${nixSystem}");

craneLibFor = p: (crane.mkLib p).overrideToolchain stableRustFor;
nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain nightlyRustFor;
craneLibFor = p: (crane.mkLib p).overrideToolchain pkgs.lre.stableRustFor;
nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain pkgs.lre.nightlyRustFor;

src = pkgs.lib.cleanSourceWith {
src = (craneLibFor pkgs).path ./.;
Expand Down Expand Up @@ -204,7 +174,7 @@
"build-chromium-tests"
./deploy/chromium-example/build_chromium_tests.sh;

docs = pkgs.callPackage ./tools/docs.nix {rust = stableRustFor pkgs;};
docs = pkgs.callPackage ./tools/docs.nix {rust = pkgs.lre.stable-rust;};

inherit (nix2container.packages.${system}.nix2container) pullImage;
inherit (nix2container.packages.${system}.nix2container) buildImage;
Expand Down Expand Up @@ -288,6 +258,14 @@
lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix {
inherit buildImage;
};
lre-rs = pkgs.callPackage ./local-remote-execution/lre-rs.nix {
inherit buildImage lre-cc;
rust-toolchains = [
pkgs.lre.stable-rust
pkgs.lre.nightly-rust
];
};

toolchain-drake = buildImage {
name = "toolchain-drake";
# imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds.
Expand Down Expand Up @@ -365,6 +343,7 @@
inherit
local-image-test
lre-cc
lre-rs
native-cli
nativelink
nativelinkCoverageForHost
Expand All @@ -376,13 +355,17 @@
nativelink-x86_64-linux
publish-ghcr
;

inherit (pkgs.lre) stable-rust nightly-rust;

default = nativelink;

rbe-autogen-lre-cc = rbe-autogen lre-cc;
nativelink-worker-lre-cc = createWorker lre-cc;
lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;};
rbe-autogen-lre-java = rbe-autogen lre-java;
nativelink-worker-lre-java = createWorker lre-java;
nativelink-worker-lre-rs = createWorker lre-rs;
nativelink-worker-siso-chromium = createWorker siso-chromium;
nativelink-worker-toolchain-drake = createWorker toolchain-drake;
nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2;
Expand Down Expand Up @@ -416,15 +399,18 @@
pre-commit.settings = {
hooks = import ./tools/pre-commit-hooks.nix {
inherit pkgs;
nightly-rust = pkgs.rust-bin.nightly.${nightly-rust-version};
nightly-rust = pkgs.rust-bin.nightly.${pkgs.lre.nightly-rust.meta.version};
};
};
local-remote-execution.settings = {
Env =
if pkgs.stdenv.isDarwin
then [] # Doesn't support Darwin yet.
else lre-cc.meta.Env;
prefix = "linux";
then lre-rs.meta.Env # C++ doesn't support Darwin yet.
else (lre-cc.meta.Env ++ lre-rs.meta.Env);
prefix =
if pkgs.stdenv.isDarwin
then "macos"
else "linux";
};
nixos.settings = {
path = with pkgs; [
Expand All @@ -449,8 +435,9 @@
pkgs.pre-commit

# Rust
(stableRustFor pkgs)
bazel
pkgs.lre.stable-rust
pkgs.lre.lre-rs

## Infrastructure
pkgs.awscli2
Expand Down
15 changes: 15 additions & 0 deletions local-remote-execution/MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,18 @@ bazel_dep(name = "rules_cc", version = "0.0.17")

# Use the starlark implementation of Java rules instead of the builtin ones.
bazel_dep(name = "rules_java", version = "8.5.1")
bazel_dep(name = "rules_rust", version = "0.54.1")
bazel_dep(name = "bazel_skylib", version = "1.7.1")

lre_rs = use_extension("//lre-rs:extension.bzl", "lre_rs")
use_repo(
lre_rs,
"lre-rs-nightly-aarch64-darwin",
"lre-rs-nightly-aarch64-linux",
"lre-rs-nightly-x86_64-darwin",
"lre-rs-nightly-x86_64-linux",
"lre-rs-stable-aarch64-darwin",
"lre-rs-stable-aarch64-linux",
"lre-rs-stable-x86_64-darwin",
"lre-rs-stable-x86_64-linux",
)
13 changes: 13 additions & 0 deletions local-remote-execution/libc/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
constraint_setting(name = "libc")

constraint_value(
name = "glibc",
constraint_setting = ":libc",
visibility = ["//visibility:public"],
)

constraint_value(
name = "musl",
constraint_setting = ":libc",
visibility = ["//visibility:public"],
)
27 changes: 27 additions & 0 deletions local-remote-execution/lre-rs.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
lib,
buildImage,
lre-cc,
rust-toolchains,
}: let
Env =
# Rust requires a functional C++ toolchain.
lre-cc.meta.Env
++ [
# This causes the rust toolchains to be available under `/nix/store/*`
# paths but not under "generic" paths like `/bin` or `/usr/bin`.
# This way we're guaranteed to use binary identical toolchains during
# local and remote execution.
"RUST=${lib.concatStringsSep ":" rust-toolchains}"
];
in
buildImage {
name = "lre-rs";
maxLayers = 100;
config = {inherit Env;};
# Passthrough so that other images can reuse the environment.
meta = {inherit Env;};

# Don't set a tag here so that the image is tagged by its derivation hash.
# tag = null;
}
Loading

0 comments on commit e686526

Please sign in to comment.