Move some tools to an externally usable overlay

flake.nix

@@ -155,16 +155,10 @@
             cargoExtraArgs = "--features enable_tokio_console";
           });
 
-        publish-ghcr = pkgs.callPackage ./tools/publish-ghcr.nix {};
-
-        local-image-test = pkgs.callPackage ./tools/local-image-test.nix {};
-
         nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;};
 
         generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {};
 
-        native-cli = pkgs.callPackage ./native-cli/default.nix {};
-
         build-chromium-tests =
           pkgs.writeShellScriptBin
           "build-chromium-tests"
@@ -209,7 +203,8 @@
 
         nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;};
 
-        createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;};
+        createWorker = pkgs.nativelink-tools.lib.createWorker self;
+
         buck2-toolchain = let
           buck2-nightly-rust-version = "2024-04-28";
           buck2-nightly-rust = pkgs.rust-bin.nightly.${buck2-nightly-rust-version};
@@ -305,6 +300,7 @@
           overlays = [
             self.overlays.lre
             (import ./tools/nixpkgs-disable-ratehammering-pulumi-tests.nix)
+            self.overlays.tools
             (import rust-overlay)
             (import ./tools/rust-overlay-cut-libsecret.nix)
           ];
@@ -316,14 +312,12 @@
           };
           native = {
             type = "app";
-            program = "${native-cli}/bin/native";
+            program = "${pkgs.nativelink-tools.native-cli}/bin/native";
           };
         };
         packages =
           rec {
             inherit
-              local-image-test
-              native-cli
               nativelink
               nativelinkCoverageForHost
               nativelink-aarch64-linux
@@ -332,9 +326,10 @@
               nativelink-is-executable-test
               nativelink-worker-init
               nativelink-x86_64-linux
-              publish-ghcr
               ;
 
+            inherit (pkgs.nativelink-tools) local-image-test publish-ghcr native-cli;
+
             default = nativelink;
 
             nativelink-worker-lre-cc = createWorker pkgs.lre.lre-cc.image;
@@ -444,13 +439,13 @@
               pkgs.playwright-test
 
               # Additional tools from within our development environment.
-              local-image-test
+              build-chromium-tests
+              docs
               generate-toolchains
-              pkgs.lre.lre-cc.lre-cc-configs-gen
               pkgs.lre.clang
-              native-cli
-              docs
-              build-chromium-tests
+              pkgs.lre.lre-cc.lre-cc-configs-gen
+              pkgs.nativelink-tools.local-image-test
+              pkgs.nativelink-tools.native-cli
             ]
             ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
               pkgs.darwin.apple_sdk.frameworks.CoreFoundation
@@ -512,6 +507,7 @@
       };
       overlays = {
         lre = import ./local-remote-execution/overlays/default.nix {inherit nix2container;};
+        tools = import ./tools/public/default.nix {inherit nix2container;};
       };
     };
 }

tools/public/create-worker.nix

@@ -0,0 +1,119 @@
+{
+  createWorker = {
+    bash,
+    buildEnv,
+    nix2container,
+    coreutils,
+    gnused,
+    lib,
+    runCommand,
+    runtimeShell,
+    self,
+  }: let
+    # A temporary directory. Note that this doesn't set any permissions. Those
+    # need to be added explicitly in the final image arguments.
+    mkTmp = runCommand "mkTmp" {} ''
+      mkdir -p $out/tmp
+    '';
+
+    # Permissions for the temporary directory.
+    mkTmpPerms = {
+      path = mkTmp;
+      regex = ".*";
+      mode = "1777";
+      uid = 0; # Owned by root.
+      gid = 0; # Owned by root.
+    };
+
+    # Enable the shebang `#!/usr/bin/env bash`.
+    mkEnvSymlink = runCommand "mkEnvSymlink" {} ''
+      mkdir -p $out/usr/bin
+      ln -s /bin/env $out/usr/bin/env
+    '';
+
+    user = "nativelink";
+    group = "nativelink";
+    uid = "1000";
+    gid = "1000";
+
+    mkUser = runCommand "mkUser" {} ''
+      mkdir -p $out/etc/pam.d
+
+      echo "root:x:0:0::/root:${runtimeShell}" > $out/etc/passwd
+      echo "${user}:x:${uid}:${gid}:::" >> $out/etc/passwd
+
+      echo "root:!x:::::::" > $out/etc/shadow
+      echo "${user}:!x:::::::" >> $out/etc/shadow
+
+      echo "root:x:0:" > $out/etc/group
+      echo "${group}:x:${gid}:" >> $out/etc/group
+
+      echo "root:x::" > $out/etc/gshadow
+      echo "${group}:x::" >> $out/etc/gshadow
+
+      cat > $out/etc/pam.d/other <<EOF
+      account sufficient pam_unix.so
+      auth sufficient pam_rootok.so
+      password requisite pam_unix.so nullok sha512
+      session required pam_unix.so
+      EOF
+
+      touch $out/etc/login.defs
+      mkdir -p $out/home/${user}
+    '';
+
+    # Set permissions for the user's home directory.
+    mkUserPerms = {
+      path = mkUser;
+      regex = "/home/${user}";
+      mode = "0755";
+      uid = lib.toInt uid;
+      gid = lib.toInt gid;
+      uname = user;
+      gname = group;
+    };
+  in
+    # Create a container image from a base image with the nativelink executable
+    # added and set as entrypoint. This allows arbitrary base images to be
+    # "enriched" with nativelink to create worker images for cloud deployments.
+    image:
+      nix2container.buildImage {
+        name = "nativelink-worker-${image.imageName}";
+        fromImage = image;
+        maxLayers = 20;
+        copyToRoot = [
+          mkUser
+          mkTmp
+          mkEnvSymlink
+          (buildEnv {
+            name = "${image.imageName}-buildEnv";
+            paths = [coreutils bash gnused];
+            pathsToLink = ["/bin"];
+          })
+        ];
+
+        perms = [
+          mkUserPerms
+          mkTmpPerms
+        ];
+
+        # Override the final image tag with the one from the base image to make
+        # the relationship between the toolchain and the worker extension more
+        # obvious.
+        tag = image.imageTag;
+
+        config = {
+          User = user;
+          WorkingDir = "/home/${user}";
+          Labels = {
+            "org.opencontainers.image.description" = "NativeLink worker generated from ${image.imageName}.";
+            "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/nativelink";
+            "org.opencontainers.image.licenses" = "Apache-2.0";
+            "org.opencontainers.image.revision" = "${self.rev or self.dirtyRev or "dirty"}";
+            "org.opencontainers.image.source" = "https://github.com/TraceMachina/nativelink";
+            "org.opencontainers.image.title" = "NativeLink worker for ${image.imageName}";
+            "org.opencontainers.image.vendor" = "Trace Machina, Inc.";
+          };
+        };
+      };
+}

tools/public/default.nix

@@ -0,0 +1,17 @@
+{nix2container}: final: _prev: {
+  inherit (nix2container.packages.${final.system}) nix2container;
+
+  # Note: Only put tools here that should be usable from external flakes.
+  nativelink-tools = {
+    local-image-test = final.callPackage ./local-image-test.nix {};
+    publish-ghcr = final.callPackage ./publish-ghcr.nix {};
+    native-cli = final.callPackage ../../native-cli/default.nix {};
+
+    lib = {
+      createWorker = self:
+        final.callPackage (import ./create-worker.nix).createWorker {
+          inherit self;
+        };
+    };
+  };
+}