diff --git a/.bazelignore b/.bazelignore index c52920b24..bf4b8b95f 100644 --- a/.bazelignore +++ b/.bazelignore @@ -13,3 +13,4 @@ bazel-testlogs bazel-nativelink local-remote-execution/generated-cc local-remote-execution/generated-java +local-remote-execution/lre-rs diff --git a/MODULE.bazel b/MODULE.bazel index b40d261fb..8a95442a1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( bazel_dep(name = "rules_cc", version = "0.0.17") bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "rules_python", version = "0.36.0") +bazel_dep(name = "rules_python", version = "0.40.0") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( @@ -19,30 +19,21 @@ python.toolchain( use_repo(python, python = "python_versions") bazel_dep(name = "rules_rust", version = "0.54.1") +archive_override( + module_name = "rules_rust", + integrity = "sha256-r09Wyq5QqZpov845sUG1Cd1oVIyCBLmKt6HK/JTVuwI=", + patch_strip = 1, + patches = ["//tools:rules_rust-musl-platforms.diff"], + urls = [ + "https://github.com/bazelbuild/rules_rust/releases/download/0.54.1/rules_rust-v0.54.1.tar.gz", + ], +) rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") rust.toolchain( edition = "2021", - rust_analyzer_version = "nightly/2024-11-23", - rustfmt_version = "nightly/2024-11-23", - sha256s = { - "2024-11-23/cargo-nightly-x86_64-apple-darwin.tar.xz": "1b0fca7da21cbbc00faef55999a2f034980abcb7da898ccdcdc12ffa18af0eef", - "2024-11-23/cargo-nightly-x86_64-unknown-linux-gnu.tar.xz": "54e155e906ca24449dcb3e8da9ef3896a4d025b28593285b5580bcb69f6bdb73", - "2024-11-23/clippy-nightly-x86_64-apple-darwin.tar.xz": "0a39d1139eb51f158861c782a884cc23ce44fe83fcb295534daf0a0f556a9e0c", - "2024-11-23/clippy-nightly-x86_64-unknown-linux-gnu.tar.xz": "b401ac1362c89a08107906fd46ce11490040eefa498d5e263e513762776e0852", - "2024-11-23/llvm-tools-nightly-x86_64-apple-darwin.tar.xz": "2d07d5a6f5b3f947e5a3ae3261f4551b007d34de5c113681d0784a8f5d9cd0e0", - "2024-11-23/llvm-tools-nightly-x86_64-unknown-linux-gnu.tar.xz": "d9b308aa7b7e76fe0fecb1ee593854690709c8b6d0e362a1479b0b0629de9191", - "2024-11-23/rust-std-nightly-x86_64-apple-darwin.tar.xz": "7d7f1a9759b28fd39929d2d7bcee99bc01b6c330f29f1acf261d307b1338c466", - "2024-11-23/rust-std-nightly-x86_64-unknown-linux-gnu.tar.xz": "502a5e7b6716add6bc9928ff841b3778bae55b07cc80dc401c57acb726b65566", - "2024-11-23/rustc-nightly-x86_64-apple-darwin.tar.xz": "3339efb0a1c4f8f25baea1a764d8bed4bfda5c17272107ae48081165caaced4c", - "2024-11-23/rustc-nightly-x86_64-unknown-linux-gnu.tar.xz": "1ac699e3942648cf3844ff3c8f03acb66ac01b221679a62619c9ed48ad4321a2", - "2024-11-23/rustfmt-nightly-x86_64-apple-darwin.tar.xz": "e67b50b801b14c2f06dd10544e66a0e76374b716bfb92f6f4b7055b4d4023ec7", - "2024-11-23/rustfmt-nightly-x86_64-unknown-linux-gnu.tar.xz": "ac2a8fff891473220d36c9e62afc2dc0d5aebaefbb8489584bd2446a37d6fd7b", - }, - versions = [ - "1.82.0", - "nightly/2024-11-23", - ], + # Rust toolchains supplied by LRE. + versions = [], ) rust_host_tools = use_extension( @@ -64,11 +55,13 @@ crate.from_cargo( supported_platform_triples = [ "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", "arm-unknown-linux-gnueabi", "armv7-unknown-linux-gnueabi", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", ], ) use_repo(crate, "crates") @@ -79,8 +72,8 @@ rust_analyzer = use_extension( ) rust_analyzer.rust_analyzer_dependencies() -bazel_dep(name = "protobuf", version = "27.5", repo_name = "com_google_protobuf") -bazel_dep(name = "toolchains_protoc", version = "0.3.3") +bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf") +bazel_dep(name = "toolchains_protoc", version = "0.3.4") protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc") protoc.toolchain( diff --git a/flake.nix b/flake.nix index 2738191ff..15b32efc2 100644 --- a/flake.nix +++ b/flake.nix @@ -52,20 +52,53 @@ system, ... }: let + # C and C++ toolchain configuration. The basis of all toolchains. + llvmPackages = pkgs.llvmPackages_19; + + # Rust toolchain configuration. stable-rust-version = "1.82.0"; nightly-rust-version = "2024-11-23"; - llvmPackages = pkgs.llvmPackages_19; + # This map translates execution platforms to sensible targets that can + # be built on such a platform. For instance, an x86_64-linux execution + # platform can target aarch64-linux and musl targets, but not darwin. + # + # On the Bazel side each key maps to a dedicated toolchain capture the + # cross-compile capabilities on the different exec platforms. + nixSystemToRustTargets = { + "aarch64-darwin" = [ + "aarch64-apple-darwin" + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-apple-darwin" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + "aarch64-linux" = [ + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + "x86_64-darwin" = [ + "aarch64-apple-darwin" + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-apple-darwin" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + "x86_64-linux" = [ + "aarch64-unknown-linux-gnu" + "aarch64-unknown-linux-musl" + "x86_64-unknown-linux-gnu" + "x86_64-unknown-linux-musl" + ]; + }; - nixSystemToRustTriple = nixSystem: - { - "x86_64-linux" = "x86_64-unknown-linux-musl"; - "aarch64-linux" = "aarch64-unknown-linux-musl"; - "x86_64-darwin" = "x86_64-apple-darwin"; - "aarch64-darwin" = "aarch64-apple-darwin"; - } - .${nixSystem} - or (throw "Unsupported Nix system: ${nixSystem}"); + nixExecToRustTargets = nixSystem: + nixSystemToRustTargets.${nixSystem} + or (throw "Unsupported Nix exec platform: ${nixSystem}"); # Calling `pkgs.pkgsCross` changes the host and target platform to the # cross-target but leaves the build platform the same as pkgs. @@ -80,24 +113,33 @@ # # For optimal cache reuse of different crosscompilation toolchains we # take our rust toolchain from the host's `pkgs` and remap the rust - # target to the target platform of the `pkgsCross` target. This lets us - # reuse the same executables (for instance rustc) to build artifacts for - # different target platforms. + # target to sensible `pkgsCross` targets. This lets us reuse the same + # executables (for instance rustc) to build artifacts for different + # target platforms. stableRustFor = p: p.rust-bin.stable.${stable-rust-version}.default.override { - targets = [ - "${nixSystemToRustTriple p.stdenv.targetPlatform.system}" - ]; + targets = nixExecToRustTargets p.stdenv.targetPlatform.system; }; nightlyRustFor = p: p.rust-bin.nightly.${nightly-rust-version}.default.override { extensions = ["llvm-tools"]; - targets = [ - "${nixSystemToRustTriple p.stdenv.targetPlatform.system}" - ]; + targets = nixExecToRustTargets p.stdenv.targetPlatform.system; }; + # This convenience method maps a nix system to the Rust target triple + # that we'd want to target by default. This is only used for Nix builds + # and doesn't have a Bazel equivalent. + nixSystemToRustTriple = nixSystem: + { + "x86_64-linux" = "x86_64-unknown-linux-musl"; + "aarch64-linux" = "aarch64-unknown-linux-musl"; + "x86_64-darwin" = "x86_64-apple-darwin"; + "aarch64-darwin" = "aarch64-apple-darwin"; + } + .${nixSystem} + or (throw "Unsupported Nix host platform: ${nixSystem}"); + craneLibFor = p: (crane.mkLib p).overrideToolchain stableRustFor; nightlyCraneLibFor = p: (crane.mkLib p).overrideToolchain nightlyRustFor; @@ -248,6 +290,9 @@ inherit (pkgs.lre) stdenv; }; createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;}; + gen-lre-rs = pkgs.callPackage ./tools/lre-rs.nix { + inherit nixSystemToRustTargets; + }; buck2-toolchain = let buck2-nightly-rust-version = "2024-04-28"; buck2-nightly-rust = pkgs.rust-bin.nightly.${buck2-nightly-rust-version}; @@ -288,6 +333,14 @@ lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix { inherit buildImage; }; + lre-rs = pkgs.callPackage ./local-remote-execution/lre-rs.nix { + inherit buildImage lre-cc; + rust-toolchains = [ + (stableRustFor pkgs) + (nightlyRustFor pkgs) + ]; + }; + toolchain-drake = buildImage { name = "toolchain-drake"; # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds. @@ -365,6 +418,7 @@ inherit local-image-test lre-cc + lre-rs native-cli nativelink nativelinkCoverageForHost @@ -376,6 +430,10 @@ nativelink-x86_64-linux publish-ghcr ; + + stable-rust = stableRustFor pkgs; + nightly-rust = nightlyRustFor pkgs; + default = nativelink; rbe-autogen-lre-cc = rbe-autogen lre-cc; @@ -383,6 +441,7 @@ lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;}; rbe-autogen-lre-java = rbe-autogen lre-java; nativelink-worker-lre-java = createWorker lre-java; + nativelink-worker-lre-rs = createWorker lre-rs; nativelink-worker-siso-chromium = createWorker siso-chromium; nativelink-worker-toolchain-drake = createWorker toolchain-drake; nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2; @@ -422,9 +481,12 @@ local-remote-execution.settings = { Env = if pkgs.stdenv.isDarwin - then [] # Doesn't support Darwin yet. - else lre-cc.meta.Env; - prefix = "linux"; + then [lre-rs.meta.Env] # C++ doesn't support Darwin yet. + else (lre-cc.meta.Env ++ lre-rs.meta.Env); + prefix = + if pkgs.stdenv.isDarwin + then "macos" + else "linux"; }; nixos.settings = { path = with pkgs; [ @@ -451,6 +513,7 @@ # Rust (stableRustFor pkgs) bazel + gen-lre-rs ## Infrastructure pkgs.awscli2 diff --git a/local-remote-execution/MODULE.bazel b/local-remote-execution/MODULE.bazel index fed578399..c0d1dcbb9 100644 --- a/local-remote-execution/MODULE.bazel +++ b/local-remote-execution/MODULE.bazel @@ -14,3 +14,18 @@ bazel_dep(name = "rules_cc", version = "0.0.17") # Use the starlark implementation of Java rules instead of the builtin ones. bazel_dep(name = "rules_java", version = "8.5.1") +bazel_dep(name = "rules_rust", version = "0.54.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") + +lre_rs = use_extension("//lre-rs:extension.bzl", "lre_rs") +use_repo( + lre_rs, + "lre-rs-nightly-aarch64-darwin", + "lre-rs-nightly-aarch64-linux", + "lre-rs-nightly-x86_64-darwin", + "lre-rs-nightly-x86_64-linux", + "lre-rs-stable-aarch64-darwin", + "lre-rs-stable-aarch64-linux", + "lre-rs-stable-x86_64-darwin", + "lre-rs-stable-x86_64-linux", +) diff --git a/local-remote-execution/libc/BUILD.bazel b/local-remote-execution/libc/BUILD.bazel new file mode 100644 index 000000000..766554cb1 --- /dev/null +++ b/local-remote-execution/libc/BUILD.bazel @@ -0,0 +1,13 @@ +constraint_setting(name = "libc") + +constraint_value( + name = "glibc", + constraint_setting = ":libc", + visibility = ["//visibility:public"], +) + +constraint_value( + name = "musl", + constraint_setting = ":libc", + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs.nix b/local-remote-execution/lre-rs.nix new file mode 100644 index 000000000..2e1055031 --- /dev/null +++ b/local-remote-execution/lre-rs.nix @@ -0,0 +1,27 @@ +{ + lib, + buildImage, + lre-cc, + rust-toolchains, +}: let + Env = + # Rust requires a functional C++ toolchain. + lre-cc.meta.Env + ++ [ + # This causes the rust toolchains to be available under `/nix/store/*` + # paths but not under "generic" paths like `/bin` or `/usr/bin`. + # This way we're guaranteed to use binary identical toolchains during + # local and remote execution. + "RUST=${lib.concatStringsSep ":" rust-toolchains}" + ]; +in + buildImage { + name = "lre-rs"; + maxLayers = 100; + config = {inherit Env;}; + # Passthrough so that other images can reuse the environment. + meta = {inherit Env;}; + + # Don't set a tag here so that the image is tagged by its derivation hash. + # tag = null; + } diff --git a/local-remote-execution/lre-rs/BUILD.bazel b/local-remote-execution/lre-rs/BUILD.bazel new file mode 100644 index 000000000..ba0cd85de --- /dev/null +++ b/local-remote-execution/lre-rs/BUILD.bazel @@ -0,0 +1,242 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load( + "@rules_rust//rust:toolchain.bzl", + "rust_toolchain", + "rustfmt_toolchain", +) + +string_flag( + name = "channel", + build_setting_default = "stable", + values = [ + "stable", + "nightly", + ], + visibility = ["//visibility:public"], +) + +config_setting( + name = "stable", + flag_values = {":channel": "stable"}, +) + +config_setting( + name = "nightly", + flag_values = {":channel": "nightly"}, +) + +string_flag( + name = "rustfmt-channel", + # It's a common use case (and matching with rules_rust's default behavior) + # to use nightly rustfmt with a stable rust toolchain, so we default to + # nightly here. + build_setting_default = "nightly", + values = [ + "stable", + "nightly", + ], + visibility = ["//visibility:public"], +) + +config_setting( + name = "rustfmt-stable", + flag_values = {":rustfmt-channel": "stable"}, +) + +config_setting( + name = "rustfmt-nightly", + flag_values = {":rustfmt-channel": "nightly"}, +) + +[ + rust_toolchain( + name = "rust-{}-{}_impl".format(channel, nix_system), + allocator_library = select({ + "@local-remote-execution//libc:musl": None, + "//conditions:default": "@rules_rust//ffi/cc/allocator_library", + }), + binary_ext = "", + cargo = "@lre-rs-{}-{}//:cargo".format(channel, nix_system), + cargo_clippy = "@lre-rs-{}-{}//:cargo-clippy".format(channel, nix_system), + clippy_driver = "@lre-rs-{}-{}//:clippy-driver".format(channel, nix_system), + debug_info = { + "dbg": "2", + "fastbuild": "0", + "opt": "0", + }, + default_edition = "2021", + dylib_ext = select({ + "@platforms//os:linux": ".so", + "@platforms//os:macos": ".dylib", + "//conditions:default": "@platforms//:incompatible", + }), + env = {}, + exec_triple = { + "x86_64-linux": "x86_64-unknown-linux-gnu", + "aarch64-linux": "aarch64-unknown-linux-gnu", + "x86_64-darwin": "x86_64-apple-darwin", + "aarch64-darwin": "aarch64-apple-darwin", + }[nix_system], + + # TODO(aaronmondal): Make these easily configurable. + extra_exec_rustc_flags = [], + extra_rustc_flags = [], + extra_rustc_flags_for_crate_types = {}, + global_allocator_library = select({ + "@local-remote-execution//libc:musl": None, + "//conditions:default": "@rules_rust//ffi/cc/global_allocator_library", + }), + + # TODO(aaronmondal): Implement these. + # llvm_cov, + # llvm_profdata, + # llvm_tools, + opt_level = { + "dbg": "0", + "fastbuild": "0", + "opt": "3", + }, + per_crate_rustc_flags = [], + rust_doc = "@lre-rs-{}-{}//:rustdoc".format(channel, nix_system), + rust_std = "@lre-rs-{}-{}//:rust_std".format(channel, nix_system), + rustc = "@lre-rs-{}-{}//:rustc".format(channel, nix_system), + rustc_lib = "@lre-rs-{}-{}//:rustc_lib".format(channel, nix_system), + rustfmt = None, # Rustfmt actions use the rustfmt-toolchain. + staticlib_ext = ".a", + stdlib_linkflags = select( + { + "@local-remote-execution//libc:glibc": [ + "-fuse-ld=mold", + "-lpthread", + "-ldl", + ], + "@local-remote-execution//libc:musl": [ + "-fuse-ld=mold", + ], + "@platforms//os:macos": [], + }, + no_match_error = "Linking for unset libc on linux isn't implemented yet.", + ), + strip_level = { + "dbg": "none", + "fastbuild": "none", + "opt": "debuginfo", + }, + target_json = "", + target_triple = select( + { + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": "aarch64-unknown-linux-gnu", + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": "aarch64-unknown-linux-musl", + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": "x86_64-unknown-linux-gnu", + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": "x86_64-unknown-linux-musl", + } | ({ + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": "x86_64-apple-darwin", + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": "aarch64-apple-darwin", + } if nix_system in [ + "aarch64-darwin", + "x86_64-darwin", + ] else {}), + no_match_error = "Incompatible target triple supplied to lre-rs-toolchain running on {}.".format(nix_system), + ), + ) + for channel in [ + "stable", + "nightly", + ] + for nix_system in [ + "x86_64-linux", + "aarch64-linux", + "x86_64-darwin", + "aarch64-darwin", + ] +] + +[ + toolchain( + name = "rust-{}-{}".format(arch, os), + exec_compatible_with = [ + "@platforms//os:{}".format(os if os == "linux" else "macos"), + "@platforms//cpu:{}".format(arch), + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = + # Linux toolchains can only target other linux systems. + # Darwin toolchains can target everything. + ["@platforms//os:linux"] if os == "linux" else [], + toolchain = select({ + ":stable": ":rust-stable-{}-{}_impl".format(arch, os), + ":nightly": ":rust-nightly-{}-{}_impl".format(arch, os), + }), + toolchain_type = "@rules_rust//rust:toolchain_type", + ) + for arch in [ + "aarch64", + "x86_64", + ] + for os in [ + "linux", + "darwin", + ] +] + +[ + rustfmt_toolchain( + name = "rustfmt-{}-{}_impl".format(channel, nix_system), + rustc = "@lre-rs-{}-{}//:rustc".format(channel, nix_system), + rustc_lib = "@lre-rs-{}-{}//:rustc_lib".format(channel, nix_system), + rustfmt = "@lre-rs-{}-{}//:rustfmt".format(channel, nix_system), + ) + for nix_system in [ + "x86_64-linux", + "aarch64-linux", + "x86_64-darwin", + "aarch64-darwin", + ] + for channel in [ + "stable", + "nightly", + ] +] + +# Limit rustfmt to exec == target until we find a good reason to do otherwise. +[ + toolchain( + name = "rustfmt-{}-{}".format(arch, os), + exec_compatible_with = [ + "@platforms//os:{}".format(os if os == "linux" else "macos"), + "@platforms//cpu:{}".format(arch), + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = [ + "@platforms//os:{}".format(os if os == "linux" else "macos"), + "@platforms//cpu:{}".format(arch), + ], + toolchain = select({ + ":rustfmt-stable": ":rustfmt-stable-{}-{}_impl".format(arch, os), + ":rustfmt-nightly": ":rustfmt-nightly-{}-{}_impl".format(arch, os), + }), + toolchain_type = "@rules_rust//rust/rustfmt:toolchain_type", + ) + for arch in [ + "aarch64", + "x86_64", + ] + for os in [ + "darwin", + "linux", + ] +] diff --git a/local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel b/local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel new file mode 100644 index 000000000..d72b70d44 --- /dev/null +++ b/local-remote-execution/lre-rs/aarch64-darwin.BUILD.bazel @@ -0,0 +1,125 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on aarch64-darwin.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/aarch64-apple-darwin/bin/rust-lld", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/x86_64-apple-darwin/bin/rust-lld", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "lib/rustlib/aarch64-apple-darwin/lib/*.rlib", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + "lib/rustlib/aarch64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "lib/rustlib/x86_64-apple-darwin/lib/*.rlib", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + "lib/rustlib/x86_64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel b/local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel new file mode 100644 index 000000000..7a5474a64 --- /dev/null +++ b/local-remote-execution/lre-rs/aarch64-linux.BUILD.bazel @@ -0,0 +1,101 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on aarch64-linux.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs/extension.bzl b/local-remote-execution/lre-rs/extension.bzl new file mode 100644 index 000000000..f40abda65 --- /dev/null +++ b/local-remote-execution/lre-rs/extension.bzl @@ -0,0 +1,71 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Module extension to register different lre-rs tool repositories.""" + +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +# TODO(aaronmondal): Using module extensions here isn't optimal as it doesn't +# allow overriding tools via environment variables. +# Unfortunately rules_rust's rust_toolchain currently +# requires tools to be declared via labels to filegroups, +# so we need the new_local_repository approach here. +# Add support for raw strings to rules_rust upstream so +# that we can remove this module extension entirely. + +def _lre_rs_impl(_mctx): + new_local_repository( + name = "lre-rs-stable-aarch64-darwin", + build_file = "@local-remote-execution//lre-rs:aarch64-darwin.BUILD.bazel", + path = "/nix/store/59m1dvvnr0imcwdd0xiqaljis9ybvy24-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-aarch64-darwin", + build_file = "@local-remote-execution//lre-rs:aarch64-darwin.BUILD.bazel", + path = "/nix/store/pjd6k6zj8blkxr551bkpz0xq3ycj9yhn-rust-default-1.85.0-nightly-2024-11-23", + ) + new_local_repository( + name = "lre-rs-stable-aarch64-linux", + build_file = "@local-remote-execution//lre-rs:aarch64-linux.BUILD.bazel", + path = "/nix/store/pyr87frpmb04bh3s5sf7vszz9855w603-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-aarch64-linux", + build_file = "@local-remote-execution//lre-rs:aarch64-linux.BUILD.bazel", + path = "/nix/store/afdl7canh5p6krqzw5vm1g4kihhqfvsr-rust-default-1.85.0-nightly-2024-11-23", + ) + new_local_repository( + name = "lre-rs-stable-x86_64-darwin", + build_file = "@local-remote-execution//lre-rs:x86_64-darwin.BUILD.bazel", + path = "/nix/store/x030cxhzj00bw41yaqiy2ihjj1mgi6mr-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-x86_64-darwin", + build_file = "@local-remote-execution//lre-rs:x86_64-darwin.BUILD.bazel", + path = "/nix/store/g1irmsn6x1gndkjisal978jfs5s0dqb2-rust-default-1.85.0-nightly-2024-11-23", + ) + new_local_repository( + name = "lre-rs-stable-x86_64-linux", + build_file = "@local-remote-execution//lre-rs:x86_64-linux.BUILD.bazel", + path = "/nix/store/kl18sr2a45vyyqv39clrjclmadacy9mc-rust-default-1.82.0", + ) + new_local_repository( + name = "lre-rs-nightly-x86_64-linux", + build_file = "@local-remote-execution//lre-rs:x86_64-linux.BUILD.bazel", + path = "/nix/store/902a168nsh4vha014j538abaji3lpwa2-rust-default-1.85.0-nightly-2024-11-23", + ) + +lre_rs = module_extension(implementation = _lre_rs_impl) diff --git a/local-remote-execution/lre-rs/platforms/BUILD.bazel b/local-remote-execution/lre-rs/platforms/BUILD.bazel new file mode 100644 index 000000000..3a926afcd --- /dev/null +++ b/local-remote-execution/lre-rs/platforms/BUILD.bazel @@ -0,0 +1,78 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +# Some toolchains, like the darwin ones, don't have an actual container image. +# We still map them to the output of the tag of the corresponding theoretical +# worker image so that bare metal metal workers can advertise exact +# exec_properties to schedulers. + +platform( + name = "aarch64-apple-darwin", + exec_properties = { + # nix eval .#packages.aarch64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": "xbwngvrnr7ygazcr20a3mddsx05l3nxn", + }, + parents = ["@local-remote-execution//nix-system:aarch64-darwin"], +) + +platform( + name = "aarch64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "j8gd39m51074dgm3d82bm68dda8lbg62", + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], +) + +platform( + name = "aarch64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "j8gd39m51074dgm3d82bm68dda8lbg62", + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], +) + +platform( + name = "x86_64-apple-darwin", + exec_properties = { + # nix eval .#packages.x86_64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": "ywhklqs9l7lz59c1l2fzqp2nzza1c75q", + }, + parents = ["@local-remote-execution//nix-system:x86_64-darwin"], +) + +platform( + name = "x86_64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "phg85ss1hq298ixjn8iqnbrhh5x5x384", + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], +) + +platform( + name = "x86_64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": "phg85ss1hq298ixjn8iqnbrhh5x5x384", + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], +) diff --git a/local-remote-execution/lre-rs/triple/BUILD.bazel b/local-remote-execution/lre-rs/triple/BUILD.bazel new file mode 100644 index 000000000..3a60ceade --- /dev/null +++ b/local-remote-execution/lre-rs/triple/BUILD.bazel @@ -0,0 +1,69 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These config settings correspond to rust triples. + +load("@bazel_skylib//lib:selects.bzl", "selects") + +selects.config_setting_group( + name = "aarch64-apple-darwin", + match_all = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], +) + +selects.config_setting_group( + name = "aarch64-unknown-linux-gnu", + match_all = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@local-remote-execution//libc:glibc", + ], +) + +selects.config_setting_group( + name = "aarch64-unknown-linux-musl", + match_all = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@local-remote-execution//libc:musl", + ], +) + +selects.config_setting_group( + name = "x86_64-apple-darwin", + match_all = [ + "@platforms//cpu:x86_64", + "@platforms//os:macos", + ], +) + +selects.config_setting_group( + name = "x86_64-unknown-linux-gnu", + match_all = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + "@local-remote-execution//libc:glibc", + ], +) + +selects.config_setting_group( + name = "x86_64-unknown-linux-musl", + match_all = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + "@local-remote-execution//libc:musl", + ], +) diff --git a/local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel b/local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel new file mode 100644 index 000000000..016833c6b --- /dev/null +++ b/local-remote-execution/lre-rs/x86_64-darwin.BUILD.bazel @@ -0,0 +1,125 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on x86_64-darwin.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/aarch64-apple-darwin/bin/rust-lld", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-apple-darwin/codegen-backends/*.so", + "lib/rustlib/x86_64-apple-darwin/bin/rust-lld", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-apple-darwin": glob([ + "lib/rustlib/aarch64-apple-darwin/lib/*.rlib", + "lib/rustlib/aarch64-apple-darwin/lib/*.so", + "lib/rustlib/aarch64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-apple-darwin": glob([ + "lib/rustlib/x86_64-apple-darwin/lib/*.rlib", + "lib/rustlib/x86_64-apple-darwin/lib/*.so", + "lib/rustlib/x86_64-apple-darwin/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel b/local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel new file mode 100644 index 000000000..000925c3e --- /dev/null +++ b/local-remote-execution/lre-rs/x86_64-linux.BUILD.bazel @@ -0,0 +1,101 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is @generated by lre-rs. + +"""Tool repository for lre-rs executing on x86_64-linux.""" + +load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + +[ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] +] + +filegroup( + name = "rustc_lib", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "bin/*.so", + "lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/codegen-backends/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/bin/rust-lld", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + ]), + }), + visibility = ["//visibility:public"], +) + +rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-gnu": glob([ + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:aarch64-unknown-linux-musl": glob([ + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.so", + "lib/rustlib/aarch64-unknown-linux-musl/lib/*.a", + "lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/**", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-gnu": glob([ + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-gnu/lib/*.a", + ]), + "@local-remote-execution//lre-rs/triple:x86_64-unknown-linux-musl": glob([ + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.rlib", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.so", + "lib/rustlib/x86_64-unknown-linux-musl/lib/*.a", + "lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/**", + ]), + }), + visibility = ["//visibility:public"], +) diff --git a/local-remote-execution/modules/lre.nix b/local-remote-execution/modules/lre.nix index 4d3f08582..61814f5a2 100644 --- a/local-remote-execution/modules/lre.nix +++ b/local-remote-execution/modules/lre.nix @@ -4,6 +4,16 @@ pkgs, ... }: let + nixExecToRustExec = nixSystem: + { + "aarch64-darwin" = "aarch64-apple-darwin"; + "aarch64-linux" = "aarch64-unknown-linux-gnu"; + "x86_64-darwin" = "x86_64-apple-darwin"; + "x86_64-linux" = "x86_64-unknown-linux-gnu"; + } + .${nixSystem} + or (throw "Unsupported Nix exec platform: ${nixSystem}"); + # These flags cause a Bazel build to use LRE toolchains regardless of whether # the build is running in local or remote configuration. # @@ -21,22 +31,66 @@ # ''; # }; # ``` - defaultConfig = [ - # TODO(aaronmondal): Remove after resolution of: - # https://github.com/bazelbuild/bazel/issues/19714#issuecomment-1745604978 - "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1" - - # TODO(aaronmondal): Remove after resolution of: - # https://github.com/bazelbuild/bazel/issues/7254 - "--define=EXECUTOR=remote" - - # Set up the default toolchains. - # TODO(aaronmondal): Implement a mechanism that autogenerates these values - # and generalizes to extensions of the lre-cc base - # toolchain (such as CUDA extensions). - "--extra_execution_platforms=@local-remote-execution//generated-cc/config:platform" - "--extra_toolchains=@local-remote-execution//generated-cc/config:cc-toolchain" - ]; + defaultConfig = + [ + # Global configuration. + + # TODO(aaronmondal): Remove after resolution of: + # https://github.com/bazelbuild/bazel/issues/7254 + "--define=EXECUTOR=remote" + ] + # C++. + # TODO(aaronmondal): At the moment lre-cc only supports x86_64-linux. + # Extend this to more nix systems. + # See: https://github.com/bazelbuild/bazel/issues/19714#issuecomment-1745604978 + ++ lib.optionals pkgs.stdenv.isLinux [ + "--action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1" + + # TODO(aaronmondal): Reimplement rbe-configs-gen for C++ so that we can + # support more execution and target platforms here. + + # Explicitly duplicate the host as an execution platform. This way local + # execution is treated the same as remote execution. + "--extra_execution_platforms=@local-remote-execution//generated-cc/config:platform" + # The C++ toolchain running on the execution platform. + "--extra_toolchains=@local-remote-execution//generated-cc/config:cc-toolchain" + + # TODO(aaronmondal): Support different target platforms in lre-cc and add + # `--platforms` settings here. + ] + # Rust. + ++ [ + # Explicitly duplicate the host as an execution platform. This way local + # execution is treated the same as remote execution. + # + # When using remote executors that differ from the host this needs to be + # manually extended via the `--extra_execution_platforms` flag. + "--extra_execution_platforms=@local-remote-execution//lre-rs/platforms:${nixExecToRustExec pkgs.system}" + + # The rust toolchains executing on the execution platform. We default to the + # platforms corresponding to the host. When using remote executors that + # differ from the platform corresponding to the host this should be extended + # via manual `--extra_toolchains` arguments. + "--extra_toolchains=@local-remote-execution//lre-rs:rust-${pkgs.system}" + "--extra_toolchains=@local-remote-execution//lre-rs:rustfmt-${pkgs.system}" + ] + # Defaults for rust target platforms. This is a convenience setting that may + # be overridden by manual `--platforms` arguments. + # + # TODO(aaronmondal): At the moment these platforms are "rust-specific". + # Generalize this to all languages. + ++ lib.optionals (pkgs.system == "aarch64-darwin") [ + "--platforms=@local-remote-execution//lre-rs/platforms:aarch64-apple-darwin" + ] + ++ lib.optionals (pkgs.system == "aarch64-linux") [ + "--platforms=@local-remote-execution//lre-rs/platforms:aarch64-unknown-linux-musl" + ] + ++ lib.optionals (pkgs.system == "x86_64-darwin") [ + "--platforms=@local-remote-execution//lre-rs/platforms:x86_64-apple-darwin" + ] + ++ lib.optionals (pkgs.system == "x86_64-linux") [ + "--platforms=@local-remote-execution//lre-rs/platforms:x86_64-unknown-linux-musl" + ]; maybeEnv = if config.Env == [] diff --git a/local-remote-execution/nix-system/BUILD.bazel b/local-remote-execution/nix-system/BUILD.bazel new file mode 100644 index 000000000..bf1c2b2c8 --- /dev/null +++ b/local-remote-execution/nix-system/BUILD.bazel @@ -0,0 +1,87 @@ +# Copyright 2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These platforms correspond to nix systems. + +# The exec_properties correspond to kubernetes properties that help RBE systems +# to determine on which nodes in a cluster the platform could realistically be +# deployed to. For instance, you'll likely always need to match the CPU and +# you'll always want to run mac platforms on mac nodes. However, you can deploy +# linux containers onto mac systems as long as the cpu architecture matches, so +# we don't constrain "kubernetes.io/os" for the linux platforms. + +# If these defaults don't work for you you can unset exec_properties by +# overriding them with empty strings in child platforms. +# See: https://bazel.build/versions/7.4.0/reference/be/platforms-and-toolchains + +# The sensible values for "kubernetes.io/arch" and "kubernetes.io/os" correspond +# to those of GOOS and GOARCH. +# See: https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63. + +# TODO(aaronmondal): Reevaluate this implementation once Bazel 8 is released +# which adds support for platform-specific flags. + +platform( + name = "aarch64-darwin", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "arm64", + "kubernetes.io/os": "darwin", + }, + visibility = ["//visibility:public"], +) + +platform( + name = "aarch64-linux", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "arm64", + }, + visibility = ["//visibility:public"], +) + +platform( + name = "x86_64-darwin", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:macos", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "amd64", + "kubernetes.io/os": "darwin", + }, + visibility = ["//visibility:public"], +) + +platform( + name = "x86_64-linux", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "kubernetes.io/arch": "amd64", + }, + visibility = ["//visibility:public"], +) diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel index 3d36b2998..19b590b5e 100644 --- a/nativelink-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -57,6 +57,10 @@ rust_doc( "--output-format=json", "-Zunstable-options", ], + target_compatible_with = select({ + "@local-remote-execution//lre-rs:nightly": [], + "//conditions:default": ["@platforms//:incompatible"], + }), ) rust_doc_test( diff --git a/tools/lre-rs.nix b/tools/lre-rs.nix new file mode 100644 index 000000000..0ff167dc3 --- /dev/null +++ b/tools/lre-rs.nix @@ -0,0 +1,219 @@ +{ + writeShellScriptBin, + lib, + nixSystemToRustTargets, +}: let + copyright = '' + # Copyright 2024 The NativeLink Authors. All rights reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # This file is @generated by lre-rs. + ''; + + systems = [ + "aarch64-darwin" + "aarch64-linux" + "x86_64-darwin" + "x86_64-linux" + ]; + channels = [ + "stable" + "nightly" + ]; + + makeRepoDefinition = system: channel: '' + ${" "} new_local_repository( + ${" "} name = "lre-rs-${channel}-${system}", + ${" "} build_file = "@local-remote-execution//lre-rs:${system}.BUILD.bazel", + ${" "} path = "$(nix eval .#packages.${system}.${channel}-rust.outPath --raw)", + ${" "} ) + ''; + + allRepoDefinitions = lib.concatStrings (lib.flatten ( + map (system: map (channel: makeRepoDefinition system channel) channels) systems + )); + + buildFile = system: let + rustTargets = nixSystemToRustTargets.${system}; + + # Generate select entry for a target + mkSelectEntry = target: '' + ${" "} "@local-remote-execution//lre-rs/triple:${target}": glob([ + ${" "} "bin/*.so", + ${" "} "lib/*.so", + ${" "} "lib/rustlib/${target}/codegen-backends/*.so", + ${" "} "lib/rustlib/${target}/bin/rust-lld", + ${" "} "lib/rustlib/${target}/lib/*.so", + ${" "} ]),''; + + # Generate stdlib select entry for a target + mkStdlibSelectEntry = target: + '' + ${" "} "@local-remote-execution//lre-rs/triple:${target}": glob([ + ${" "} "lib/rustlib/${target}/lib/*.rlib", + ${" "} "lib/rustlib/${target}/lib/*.so", + ${" "} "lib/rustlib/${target}/lib/*.a",'' + + (lib.optionalString (builtins.match ".*-musl" target != null) + ''${"\n "} "lib/rustlib/${target}/lib/self-contained/**",'') + + ''${"\n "} ]),''; + + selectEntries = builtins.concatStringsSep "\n" (map mkSelectEntry rustTargets); + stdlibSelectEntries = builtins.concatStringsSep "\n" (map mkStdlibSelectEntry rustTargets); + in '' + ${copyright} + """Tool repository for lre-rs executing on ${system}.""" + + load("@rules_rust//rust:toolchain.bzl", "rust_stdlib_filegroup") + + [ + filegroup( + name = tool, + srcs = [ + "bin/{}".format(tool), + ], + visibility = ["//visibility:public"], + ) + for tool in [ + "rustc", + "rustfmt", + "cargo", + "cargo-clippy", + "clippy-driver", + "rustdoc", + ] + ] + + filegroup( + name = "rustc_lib", + srcs = select({ + ${selectEntries} + }), + visibility = ["//visibility:public"], + ) + + rust_stdlib_filegroup( + name = "rust_std", + srcs = select({ + ${stdlibSelectEntries} + }), + visibility = ["//visibility:public"], + )''; +in + writeShellScriptBin "lre-rs" '' + set -euo pipefail + + SRC_ROOT=$(git rev-parse --show-toplevel)/local-remote-execution + + cd "''${SRC_ROOT}" + + cat > ''${SRC_ROOT}/lre-rs/extension.bzl << EOF + ${copyright} + """Module extension to register different lre-rs tool repositories.""" + + load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + + # TODO(aaronmondal): Using module extensions here isn't optimal as it doesn't + # allow overriding tools via environment variables. + # Unfortunately rules_rust's rust_toolchain currently + # requires tools to be declared via labels to filegroups, + # so we need the new_local_repository approach here. + # Add support for raw strings to rules_rust upstream so + # that we can remove this module extension entirely. + + def _lre_rs_impl(_mctx): + ${allRepoDefinitions} + lre_rs = module_extension(implementation = _lre_rs_impl) + EOF + + cat > ''${SRC_ROOT}/lre-rs/platforms/BUILD.bazel << EOF + ${copyright} + # Some toolchains, like the darwin ones, don't have an actual container image. + # We still map them to the output of the tag of the corresponding theoretical + # worker image so that bare metal metal workers can advertise exact + # exec_properties to schedulers. + + platform( + name = "aarch64-apple-darwin", + exec_properties = { + # nix eval .#packages.aarch64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.aarch64-darwin.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:aarch64-darwin"], + ) + + platform( + name = "aarch64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], + ) + + platform( + name = "aarch64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.aarch64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:aarch64-linux"], + ) + + platform( + name = "x86_64-apple-darwin", + exec_properties = { + # nix eval .#packages.x86_64-darwin.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.x86_64-darwin.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:x86_64-darwin"], + ) + + platform( + name = "x86_64-unknown-linux-gnu", + constraint_values = ["@local-remote-execution//libc:glibc"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], + ) + + platform( + name = "x86_64-unknown-linux-musl", + constraint_values = ["@local-remote-execution//libc:musl"], + exec_properties = { + # nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag + "lre-rs": $(nix eval .#packages.x86_64-linux.nativelink-worker-lre-rs.imageTag), + }, + parents = ["@local-remote-execution//nix-system:x86_64-linux"], + ) + EOF + + cat > ''${SRC_ROOT}/lre-rs/aarch64-darwin.BUILD.bazel << EOF + ${buildFile "aarch64-darwin"} + EOF + + cat > ''${SRC_ROOT}/lre-rs/aarch64-linux.BUILD.bazel << EOF + ${buildFile "aarch64-linux"} + EOF + + cat > ''${SRC_ROOT}/lre-rs/x86_64-darwin.BUILD.bazel << EOF + ${buildFile "x86_64-darwin"} + EOF + + cat > ''${SRC_ROOT}/lre-rs/x86_64-linux.BUILD.bazel << EOF + ${buildFile "x86_64-linux"} + EOF'' diff --git a/tools/rules_rust-musl-platforms.diff b/tools/rules_rust-musl-platforms.diff new file mode 100644 index 000000000..e7429170c --- /dev/null +++ b/tools/rules_rust-musl-platforms.diff @@ -0,0 +1,13 @@ +diff --git a/rust/platform/triple_mappings.bzl b/rust/platform/triple_mappings.bzl +index b6191056..1261b529 100644 +--- a/rust/platform/triple_mappings.bzl ++++ b/rust/platform/triple_mappings.bzl +@@ -66,6 +66,8 @@ SUPPORTED_T2_PLATFORM_TRIPLES = { + "x86_64-unknown-freebsd": _support(std = True, host_tools = True), + "x86_64-unknown-fuchsia": _support(std = True, host_tools = False), + "x86_64-unknown-none": _support(std = True, host_tools = False), ++ "x86_64-unknown-linux-musl": _support(std = True, host_tools = False), ++ "aarch64-unknown-linux-musl": _support(std = True, host_tools = False), + } + + _T3_PLATFORM_TRIPLES = {