Support the MacOS sandbox for larger projects

Fixes ipetkov#482.

MacOS has trouble with derivations which (directly or transitively)
have many buildInputs.

Crane at present creates a build structure in which a given cargo
command will transitively depend on numCrates nix store paths. This
means that Crane fails to build projects with over about 600 crate
dependencies on MacOS if the sandbox is enabled.

This MR utilises a tiering approach to improve this.

Each registry is assigned to a shard based on the hash of the crate
name. If there are <32 crates in a registry there is one shard, if
<2048 there are 16 shards, otherwise 256. Crates are directly extracted
into these shard derivations rather than symlinking.

What this means is:

1. Crane will not create a vendoring derivation with many inputs
   unless a project has a truly crazy number of dependencies.
1. No downstream cargo derivation will have many inputs either.

docs/API.md

@@ -859,22 +859,24 @@ craneLib.devShell {
 }
 ```
 
-### `craneLib.downloadCargoPackage`
+### `craneLib.downloadCargoPackages`
 
-`downloadCargoPackage :: set -> drv`
+`downloadCargoPackages :: set -> drv`
 
-Download a packaged cargo crate (e.g. from crates.io) and prepare it for
-vendoring.
+Downloads a collection of cargo crates (e.g. from crates.io) and extract
+into a single directory.
 
 The registry's `fetchurlExtraArgs` will be passed through to `fetchurl` when
 downloading the crate, making it possible to influence interacting with the
 registry's API if necessary.
 
 #### Required input attributes
-* `checksum`: the (sha256) checksum recorded in the Cargo.lock file
-* `name`: the name of the crate
-* `source`: the source key recorded in the Cargo.lock file
-* `version`: the version of the crate
+* `shard`: an identifier for the shard. This will become a part of the derivation name.
+* `packages`: a list of attribute sets, each one containing:
+  * `checksum`: the (sha256) checksum recorded in the Cargo.lock file
+  * `name`: the name of the crate
+  * `source`: the source key recorded in the Cargo.lock file
+  * `version`: the version of the crate
 
 ### `craneLib.downloadCargoPackageFromGit`
 

lib/default.nix

@@ -43,7 +43,7 @@ in
     indexUrl = "https://github.com/rust-lang/crates.io-index";
   };
 
-  downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
+  downloadCargoPackages = callPackage ./downloadCargoPackages.nix { };
   downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
   filterCargoSources = callPackage ./filterCargoSources.nix { };
   findCargoFiles = callPackage ./findCargoFiles.nix { };

lib/downloadCargoPackages.nix

@@ -0,0 +1,35 @@
+{ fetchurl
+, urlForCargoPackage
+, runCommand
+, lib
+}:
+
+{shardName, packages}:
+let
+  getTarball = ({ name, version, checksum, ...}@args: let
+    pkgInfo = urlForCargoPackage args;
+    in
+    {
+      inherit name version checksum;
+      tarball = fetchurl (pkgInfo.fetchurlExtraArgs // {
+        inherit (pkgInfo) url;
+        name = "${name}-${version}";
+        sha256 = checksum;
+      });
+    });
+
+  tarballs = map getTarball packages;
+
+  extract = (tarball: let outPath = "$out/${lib.escapeShellArg "${tarball.name}-${tarball.version}"}";
+    in ''
+    mkdir -p ${outPath}
+    tar -xf ${tarball.tarball} -C ${outPath} --strip-components=1
+    echo '{"files":{}, "package":"${tarball.checksum}"}' > ${outPath}/.cargo-checksum.json
+    '');
+
+  name = if builtins.stringLength shardName == 0 then "extract-cargo-packages" else "extract-cargo-packages-${shardName}";
+in
+runCommand name { } ''
+  mkdir -p $out
+  ${lib.strings.concatMapStrings extract tarballs}
+''

lib/vendorCargoRegistries.nix

@@ -1,4 +1,4 @@
-{ downloadCargoPackage
+{ downloadCargoPackages
 , lib
 , runCommandLocal
 }:
@@ -44,12 +44,20 @@ let
     lockPackages;
   lockedRegistryGroups = groupBy (p: p.source) lockedPackagesFromRegistry;
 
-  vendorSingleRegistry = packages: runCommandLocal "vendor-registry" { } ''
-    mkdir -p $out
-    ${concatMapStrings (p: ''
-      ln -s ${escapeShellArg (downloadCargoPackage p)} $out/${escapeShellArg "${p.name}-${p.version}"}
-    '') packages}
-  '';
+  vendorSingleRegistry = (packages: let
+    packageCount = builtins.length packages;
+    shardCharacters = if packageCount < 32 then 0 else if packageCount < 2048 then 1 else 2;
+    shard = package: builtins.substring 0 shardCharacters (builtins.hashString "sha256" package.name);
+    grouped = builtins.groupBy shard packages;
+    vendoredShards = builtins.mapAttrs (shardName: packages: downloadCargoPackages {inherit shardName packages;}) grouped;
+    vendoredShardsList = builtins.attrValues vendoredShards;
+    in runCommandLocal "vendor-registry" { } ''
+      mkdir -p $out
+      ${concatMapStrings (p: ''
+      for dir in ${p}/*/; do ln -s "$(realpath "$dir")" "$out/''${dir##*/}"; done
+      '') vendoredShardsList}
+    ''
+  );
 
   parsedCargoConfigTomls = map (p: builtins.fromTOML (readFile p)) cargoConfigs;
   allCargoRegistries = flatten (map (c: c.registries or [ ]) parsedCargoConfigTomls);