From 1cc11ed9e66f59984128315dd4c841766447d759 Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Mon, 9 Dec 2024 15:47:23 +0100 Subject: [PATCH] Move lre-cc into the lre overlay This change makes lre-cc more portable and provides the structure into which we can add additional toolchains, like the upcoming lre-rs. The lre-cc logic now automatically registers the "implementation detail" packages automatically. The new command to regenerate lre-cc is now called `lre-cc`. --- flake.nix | 22 ++--- kubernetes/overlays/lre/worker-lre-cc.yaml | 2 +- local-remote-execution/flake-module.nix | 8 ++ local-remote-execution/lre-cc.nix | 37 -------- local-remote-execution/overlays/default.nix | 12 ++- local-remote-execution/overlays/lre-cc.nix | 84 +++++++++++++++++++ .../{ => overlays}/rbe-autogen.nix | 4 +- .../rbe-configs-gen/default.nix} | 0 .../rbe_configs_gen_adjustments.diff | 0 tools/generate-toolchains.nix | 29 ------- 10 files changed, 112 insertions(+), 86 deletions(-) delete mode 100644 local-remote-execution/lre-cc.nix create mode 100644 local-remote-execution/overlays/lre-cc.nix rename local-remote-execution/{ => overlays}/rbe-autogen.nix (98%) rename local-remote-execution/{rbe-configs-gen.nix => overlays/rbe-configs-gen/default.nix} (100%) rename local-remote-execution/{ => overlays/rbe-configs-gen}/rbe_configs_gen_adjustments.diff (100%) diff --git a/flake.nix b/flake.nix index 151bd6e6e..f4db47604 100644 --- a/flake.nix +++ b/flake.nix @@ -193,9 +193,7 @@ nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;}; - rbe-configs-gen = pkgs.callPackage ./local-remote-execution/rbe-configs-gen.nix {}; - - generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {inherit rbe-configs-gen;}; + generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {}; native-cli = pkgs.callPackage ./native-cli/default.nix {}; @@ -243,10 +241,6 @@ nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;}; - rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix { - inherit buildImage; - inherit (pkgs.lre) stdenv; - }; createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;}; buck2-toolchain = let buck2-nightly-rust-version = "2024-04-28"; @@ -285,9 +279,6 @@ os = "linux"; }; }; - lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix { - inherit buildImage; - }; toolchain-drake = buildImage { name = "toolchain-drake"; # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds. @@ -364,7 +355,6 @@ rec { inherit local-image-test - lre-cc native-cli nativelink nativelinkCoverageForHost @@ -378,10 +368,9 @@ ; default = nativelink; - rbe-autogen-lre-cc = rbe-autogen lre-cc; - nativelink-worker-lre-cc = createWorker lre-cc; + nativelink-worker-lre-cc = createWorker pkgs.lre.lre-cc.image; lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;}; - rbe-autogen-lre-java = rbe-autogen lre-java; + rbe-autogen-lre-java = pkgs.rbe-autogen lre-java; nativelink-worker-lre-java = createWorker lre-java; nativelink-worker-siso-chromium = createWorker siso-chromium; nativelink-worker-toolchain-drake = createWorker toolchain-drake; @@ -420,7 +409,7 @@ }; }; local-remote-execution.settings = { - Env = + Env = with pkgs.lre; if pkgs.stdenv.isDarwin then [] # Doesn't support Darwin yet. else lre-cc.meta.Env; @@ -483,6 +472,7 @@ # Additional tools from within our development environment. local-image-test generate-toolchains + pkgs.lre.lre-cc.lre-cc-configs-gen pkgs.lre.clang native-cli docs @@ -547,7 +537,7 @@ nixos = ./tools/nixos/flake-module.nix; }; overlays = { - lre = import ./local-remote-execution/overlays/default.nix; + lre = import ./local-remote-execution/overlays/default.nix {inherit nix2container;}; }; }; } diff --git a/kubernetes/overlays/lre/worker-lre-cc.yaml b/kubernetes/overlays/lre/worker-lre-cc.yaml index 46ba6b646..6363d91ec 100644 --- a/kubernetes/overlays/lre/worker-lre-cc.yaml +++ b/kubernetes/overlays/lre/worker-lre-cc.yaml @@ -25,7 +25,7 @@ spec: args: - | git config --global --add safe.directory "*" - NATIVELINK_WORKER_PLATFORM=docker://lre-cc:$(nix eval /mnt/src_root#lre-cc.imageTag --raw) && + NATIVELINK_WORKER_PLATFORM=docker://lre-cc:$(nix eval /mnt/src_root#nativelink-worker-lre-cc.imageTag --raw) && printf '#!/bin/sh\nexport NATIVELINK_WORKER_PLATFORM=%s\nexec "$@"' "$NATIVELINK_WORKER_PLATFORM" > /entrypoint/entrypoint.sh && chmod +x /entrypoint/entrypoint.sh volumeMounts: diff --git a/local-remote-execution/flake-module.nix b/local-remote-execution/flake-module.nix index 6fdda93de..2741b3d55 100644 --- a/local-remote-execution/flake-module.nix +++ b/local-remote-execution/flake-module.nix @@ -45,4 +45,12 @@ } ); }; + + config = { + perSystem = {pkgs, ...}: { + packages = { + rbe-autogen-lre-cc = pkgs.rbe-autogen pkgs.lre.lre-cc.image; + }; + }; + }; } diff --git a/local-remote-execution/lre-cc.nix b/local-remote-execution/lre-cc.nix deleted file mode 100644 index a74ed385b..000000000 --- a/local-remote-execution/lre-cc.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ - buildImage, - lre, - lib, - coreutils, - findutils, - gnutar, -}: let - # This environment is shared between toolchain autogen images and the final - # toolchain image. - Env = [ - # Add all tooling here so that the generated toolchains use `/nix/store/*` - # paths instead of `/bin` or `/usr/bin`. This way we're guaranteed to use - # binary identical toolchains during local and remote execution. - ("PATH=" - + (lib.strings.concatStringsSep ":" [ - "${lre.stdenv.cc.bintools}/bin" - "${lre.clang}/bin" - "${lre.stdenv}/bin" - "${coreutils}/bin" - "${findutils}/bin" - "${gnutar}/bin" - ])) - - "CC=${lre.clang}/bin/customClang" - ]; -in - buildImage { - name = "lre-cc"; - maxLayers = 100; - config = {inherit Env;}; - # Attached for passthrough to rbe-configs-gen. - meta = {inherit Env;}; - - # Don't set a tag here so that the image is tagged by its derivation hash. - # tag = null; - } diff --git a/local-remote-execution/overlays/default.nix b/local-remote-execution/overlays/default.nix index 3595a96fd..875fb2f73 100644 --- a/local-remote-execution/overlays/default.nix +++ b/local-remote-execution/overlays/default.nix @@ -1,4 +1,12 @@ -final: _prev: { +{nix2container}: final: _prev: { + inherit (nix2container.packages.${final.system}) nix2container; + + rbe-configs-gen = final.callPackage ./rbe-configs-gen {}; + + rbe-autogen = final.callPackage ./rbe-autogen.nix { + inherit (final.lre) stdenv; + }; + lre = { stdenv = final.callPackage ./stdenv.nix { llvmPackages = final.llvmPackages_19; @@ -8,5 +16,7 @@ final: _prev: { clang = final.callPackage ./clang.nix { inherit (final.lre) stdenv; }; + + lre-cc = final.callPackage ./lre-cc.nix {}; }; } diff --git a/local-remote-execution/overlays/lre-cc.nix b/local-remote-execution/overlays/lre-cc.nix new file mode 100644 index 000000000..8f2ca21b7 --- /dev/null +++ b/local-remote-execution/overlays/lre-cc.nix @@ -0,0 +1,84 @@ +{ + nix2container, + lre, + lib, + coreutils, + findutils, + gnutar, + bazel_7, + writeShellScriptBin, + rbe-configs-gen, +}: let + lre-cc-configs-gen = writeShellScriptBin "lre-cc" '' + set -xeuo pipefail + + SRC_ROOT=$(git rev-parse --show-toplevel)/local-remote-execution + + cd "''${SRC_ROOT}" + + LRE_CC_IMAGE_TAG=${lre.lre-cc.image.imageTag}; + + nix run .#rbe-autogen-lre-cc.copyTo \ + docker-daemon:rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} -L + + ${rbe-configs-gen}/bin/rbe_configs_gen \ + --toolchain_container=rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} \ + --exec_os=linux \ + --target_os=linux \ + --bazel_version=${bazel_7.version} \ + --output_src_root=''${SRC_ROOT} \ + --output_config_path=generated-cc \ + --generate_java_configs=false \ + --generate_cpp_configs=true \ + --bazel_path=${bazel_7}/bin/bazel \ + --cpp_env_json=cpp_env.json + + # The rbe_configs_gen tool automatically sets the exec_properties of the + # generated platform to the generator container name and tag. For efficiency + # reasons the actual deployment won't be the same as this generator + # container, so we modify this in the generated configuration. + sed -i \ + 's|rbe-autogen-lre-cc|lre-cc|g' \ + ''${SRC_ROOT}/generated-cc/config/BUILD + + chmod 644 \ + ''${SRC_ROOT}/generated-cc/LICENSE \ + ''${SRC_ROOT}/generated-cc/config/BUILD \ + + pre-commit run -a + ''; + + # TODO(aaronmondal): Move the generator logic into this packageset. + # This environment is shared between toolchain autogen images and the final + # toolchain image. + Env = [ + # Add all tooling here so that the generated toolchains use `/nix/store/*` + # paths instead of `/bin` or `/usr/bin`. This way we're guaranteed to use + # binary identical toolchains during local and remote execution. + ("PATH=" + + (lib.strings.concatStringsSep ":" [ + "${lre.stdenv.cc.bintools}/bin" + "${lre.clang}/bin" + "${lre.stdenv}/bin" + "${coreutils}/bin" + "${findutils}/bin" + "${gnutar}/bin" + ])) + + "CC=${lre.clang}/bin/customClang" + ]; + + image = nix2container.buildImage { + name = "lre-cc"; + maxLayers = 100; + config = {inherit Env;}; + # Attached for passthrough to rbe-configs-gen. + meta = {inherit Env;}; + + # Don't set a tag here so that the image is tagged by its derivation hash. + # tag = null; + }; +in { + inherit lre-cc-configs-gen image; + meta = {inherit Env;}; +} diff --git a/local-remote-execution/rbe-autogen.nix b/local-remote-execution/overlays/rbe-autogen.nix similarity index 98% rename from local-remote-execution/rbe-autogen.nix rename to local-remote-execution/overlays/rbe-autogen.nix index 9af521466..9332c855f 100644 --- a/local-remote-execution/rbe-autogen.nix +++ b/local-remote-execution/overlays/rbe-autogen.nix @@ -2,7 +2,7 @@ bash, bazel_7, buildEnv, - buildImage, + nix2container, cacert, coreutils, findutils, @@ -96,7 +96,7 @@ }; in image: - buildImage { + nix2container.buildImage { name = "autogen-${image.imageName}"; fromImage = image; maxLayers = 20; diff --git a/local-remote-execution/rbe-configs-gen.nix b/local-remote-execution/overlays/rbe-configs-gen/default.nix similarity index 100% rename from local-remote-execution/rbe-configs-gen.nix rename to local-remote-execution/overlays/rbe-configs-gen/default.nix diff --git a/local-remote-execution/rbe_configs_gen_adjustments.diff b/local-remote-execution/overlays/rbe-configs-gen/rbe_configs_gen_adjustments.diff similarity index 100% rename from local-remote-execution/rbe_configs_gen_adjustments.diff rename to local-remote-execution/overlays/rbe-configs-gen/rbe_configs_gen_adjustments.diff diff --git a/tools/generate-toolchains.nix b/tools/generate-toolchains.nix index 190c960c4..1ffaa4898 100644 --- a/tools/generate-toolchains.nix +++ b/tools/generate-toolchains.nix @@ -10,35 +10,6 @@ writeShellScriptBin "generate-toolchains" '' cd "''${SRC_ROOT}" - LRE_CC_IMAGE_TAG=$(nix eval .#lre-cc.imageTag --raw) - - nix run .#rbe-autogen-lre-cc.copyTo \ - docker-daemon:rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} -L - - ${rbe-configs-gen}/bin/rbe_configs_gen \ - --toolchain_container=rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} \ - --exec_os=linux \ - --target_os=linux \ - --bazel_version=${bazel_7.version} \ - --output_src_root=''${SRC_ROOT} \ - --output_config_path=generated-cc \ - --generate_java_configs=false \ - --generate_cpp_configs=true \ - --bazel_path=${bazel_7}/bin/bazel \ - --cpp_env_json=cpp_env.json - - # The rbe_configs_gen tool automatically sets the exec_properties of the - # generated platform to the generator container name and tag. For efficiency - # reasons the actual deployment won't be the same as this generator - # container, so we modify this in the generated configuration. - sed -i \ - 's|rbe-autogen-lre-cc|lre-cc|g' \ - ''${SRC_ROOT}/generated-cc/config/BUILD - - chmod 644 \ - ''${SRC_ROOT}/generated-cc/LICENSE \ - ''${SRC_ROOT}/generated-cc/config/BUILD \ - LRE_JAVA_IMAGE_TAG=$(nix eval .#lre-java.imageTag --raw) nix run .#rbe-autogen-lre-java.copyTo \