Much faster symlinking, safer interactions with rustc

CHANGELOG.md

@@ -18,8 +18,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   `Cargo.lock` is exactly what is expected (without implicit changes at build
   time) but this may end up rejecting builds which were previously passing. To
   get the old behavior back, set `cargoExtraArgs = "";`
-* **Breaking**: `cargoDoc` will no longer install cargo artifacts by default.
-  Set `doInstallCargoArtifacts = true;` to get the old behavior back.
+* **Breaking**: `cargoDoc` and `cargoClippy` will no longer install cargo artifacts
+  by default. Set `doInstallCargoArtifacts = true;` to get the old behavior back.
+* **Breaking**: rustc is now executed with a wrapper script. Therefore providing
+  your own value of `RUSTC_WRAPPER` will no longer work. We do not believe that
+  this is commonly used.
+* Large performance improvement when symlinking artifacts.
 * `cargoDoc` will now install generated documentation in `$out/share/doc`
 * Fixed a bug when testing proc macro crates with `cargoNextest` on macOS.
   ([#376](https://github.com/ipetkov/crane/pull/376))

docs/API.md

@@ -1439,17 +1439,16 @@ identical files against a directory of previously prepared cargo artifacts.
 It takes three positional arguments:
 1. the installation directory for the output.
    * An error will be raised if not specified
-   * If the specified path is a directory which exists then the current cargo
-     artifacts will be compared with the contents of said directory. Any files
-     whose contents and paths match will be symbolically linked together to
-     reduce the size of the data stored in the Nix store.
 1. the path to cargo's artifact directory
    * An error will be raised if not specified
 1. a path to the previously prepared cargo artifacts
    * An error will be raised if not specified
    * `/dev/null` can be specified here if there is no previous directory to
      deduplicate against
 
+'symlinks.tar' will be created in the output directory in this case. It contains
+symlinks to all of the dep files in this output, simplifying the untar behaviour.
+
 Defines `prepareAndInstallCargoArtifactsDir()` which handles installing cargo's
 artifact directory to the derivation's output. It takes three positional
 arguments:

lib/cargoClippy.nix

@@ -19,5 +19,7 @@ mkCargoDerivation (args // {
 
   buildPhaseCargoCommand = "cargoWithProfile clippy ${cargoExtraArgs} ${cargoClippyExtraArgs}";
 
+  doInstallCargoArtifacts = false;
+
   nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ clippy ];
 })

lib/mkCargoDerivation.nix

@@ -8,6 +8,7 @@
 , rsync
 , stdenv
 , vendorCargoDeps
+, writeShellApplication
 , zstd
 }:
 
@@ -39,13 +40,55 @@ let
     "pnameSuffix"
     "stdenv"
   ];
+
+  rustcWrapper = writeShellApplication {
+    name = "rustc-wrapper";
+    text = ''
+      set -euo pipefail
+
+      args=("$@")
+
+      temp_dir=$(mktemp -d)
+      trap 'rm -rf -- "$temp_dir"' EXIT
+
+      for i in "''${!args[@]}"; do
+        if [[ "''${args[i]}" == --out-dir=* ]]; then
+          current_out_dir="''${args[i]#--out-dir=}"
+          args[i]="--out-dir=$temp_dir"
+          break
+        elif [[ "''${args[i]}" == --out-dir ]]; then
+          current_out_dir="''${args[i+1]}"
+          args[i+1]="$temp_dir"
+          break
+        fi
+      done
+
+      if [[ -v current_out_dir ]]; then
+        stderr="$temp_dir/crane_stderr"
+        set +e
+        stdout=$("''${args[@]}" 2>"$stderr")
+        exit_code=$?
+        set -e
+        stderr_text=$(cat "$stderr")
+        rm "$stderr"
+        cp -r "$temp_dir/." "$current_out_dir"
+        echo -n "$stdout"
+        echo -n "$stderr_text" >&2
+        exit $exit_code
+      else
+        exec "$@"
+      fi
+    '';
+  };
 in
 chosenStdenv.mkDerivation (cleanedArgs // {
   inherit cargoArtifacts;
 
   pname = "${args.pname or crateName.pname}${args.pnameSuffix or ""}";
   version = args.version or crateName.version;
 
+  RUSTC_WRAPPER="${rustcWrapper}/bin/rustc-wrapper";
+
   # Controls whether cargo's `target` directory should be copied as an output
   doInstallCargoArtifacts = args.doInstallCargoArtifacts or true;
 

lib/setupHooks/inheritCargoArtifactsHook.sh

@@ -50,10 +50,9 @@ inheritCargoArtifacts() {
       # being built (since changing `Cargo.lock` would rebuild everything anyway), which makes them
       # good candidates for symlinking (esp. since they can make up 60-70% of the artifact directory
       # on most projects). Thus we ignore them when copying all other artifacts below as we will
-      # symlink them afterwards. Note that we scope these checks to the `/deps` subdirectory; the
-      # workspace's own .rlib and .rmeta files appear one directory up (and these may require being
-      # writable depending on how the actual workspace build is being invoked, so we'll leave them
-      # alone).
+      # symlink them afterwards. Note that we scope these checks to the `/deps` subdirectory, as
+      # this directory contains only rustc output. We have a rustc wrapper which ensures that any
+      # writes to these symlinks do not try to change the (immutable) symlink targets.
       #
       # NB: keep the executable bit only if set on the original file
       # but make all files writable as sometimes read-only files will make the build choke
@@ -66,27 +65,15 @@ inheritCargoArtifacts() {
         --times \
         --chmod=u+w \
         --executability \
-        --exclude 'deps/*.rlib' \
-        --exclude 'deps/*.rmeta' \
+        --exclude '*/deps/*' \
         --exclude '.cargo-lock' \
         "${preparedArtifacts}/" \
         "${cargoTargetDir}/"
-
-      local linkCandidates=$(mktemp linkCandidatesXXXX.txt)
-      find "${preparedArtifacts}" \
-        '(' -path '*/deps/*.rlib' -or -path '*/deps/*.rmeta' ')' \
-        -printf "%P\n" \
-        >"${linkCandidates}"
-
-      # Next create any missing directories up front so we can avoid redundant checks later
-      cat "${linkCandidates}" \
-        | xargs --no-run-if-empty -n1 dirname \
-        | sort -u \
-        | (cd "${cargoTargetDir}"; xargs --no-run-if-empty mkdir -p)
-
-      # Lastly do the actual symlinking
-      cat "${linkCandidates}" \
-        | xargs -P ${NIX_BUILD_CORES} -I '##{}##' ln -s "${preparedArtifacts}/##{}##" "${cargoTargetDir}/##{}##"
+
+      symlinks="${cargoArtifacts}/symlinks.tar"
+      if [[ -f "${symlinks}" ]]; then
+        tar -xf "${symlinks}" -C "${cargoTargetDir}" --strip-components=1
+      fi
     fi
   else
     echo unable to copy cargo artifacts, \"${preparedArtifacts}\" looks invalid

lib/setupHooks/installCargoArtifactsHook.sh

@@ -25,23 +25,16 @@ dedupAndInstallCargoArtifactsDir() {
 
   mkdir -p "${dest}"
 
-  if [ -d "${prevCargoTargetDir}" ]; then
-    echo "symlinking duplicates in ${cargoTargetDir} to ${prevCargoTargetDir}"
-
-    while read -r fullTargetFile; do
-      # Strip the common prefix of the current target directory
-      local targetFile="${fullTargetFile#"${cargoTargetDir}"}"
-      # Join the path and ensure we don't have a duplicate `/` separator
-      local candidateOrigFile="${prevCargoTargetDir}/${targetFile#/}"
-
-      if cmp --silent "${candidateOrigFile}" "${fullTargetFile}"; then
-        ln --symbolic --force --logical "${candidateOrigFile}" "${fullTargetFile}"
-      fi
-    done < <(find "${cargoTargetDir}" -type f)
-  fi
-
   echo installing "${cargoTargetDir}" to "${dest}"
   mv "${cargoTargetDir}" --target-directory="${dest}"
+
+  local symlinksDir="$(mktemp -d)"
+  cp -Rs "${dir}/target" "${symlinksDir}"
+  if test -n "$(shopt -s nullglob; echo $symlinksDir/target/*/deps)"; then
+    pushd "$symlinksDir"
+    tar -cf "${dir}/symlinks.tar" target/*/deps
+    popd
+  fi
 }
 
 prepareAndInstallCargoArtifactsDir() {