From c09d0773f6ecafdf4b182011e6071241a9d0f97f Mon Sep 17 00:00:00 2001 From: "james.baker@helsing.ai" Date: Fri, 29 Dec 2023 16:01:23 +0000 Subject: [PATCH 1/3] Support the MacOS sandbox for larger projects Fixes #482. MacOS has trouble with derivations which (directly or transitively) have many buildInputs. Crane at present creates a build structure in which a given cargo command will transitively depend on numCrates nix store paths. This means that Crane fails to build projects with over about 600 crate dependencies on MacOS if the sandbox is enabled. This MR utilises a tiering approach to improve this. Each registry is assigned to a shard based on the hash of the crate name. If there are <32 crates in a registry there is one shard, if <2048 there are 16 shards, otherwise 256. Crates are directly extracted into these shard derivations rather than symlinking. What this means is: 1. Crane will not create a vendoring derivation with many inputs unless a project has a truly crazy number of dependencies. 1. No downstream cargo derivation will have many inputs either. --- docs/API.md | 18 ++++++++++-------- lib/default.nix | 2 +- lib/downloadCargoPackage.nix | 23 ----------------------- lib/downloadCargoPackages.nix | 35 +++++++++++++++++++++++++++++++++++ lib/vendorCargoRegistries.nix | 22 +++++++++++++++------- 5 files changed, 61 insertions(+), 39 deletions(-) delete mode 100644 lib/downloadCargoPackage.nix create mode 100644 lib/downloadCargoPackages.nix diff --git a/docs/API.md b/docs/API.md index bb5cb01c..1729c443 100644 --- a/docs/API.md +++ b/docs/API.md @@ -859,22 +859,24 @@ craneLib.devShell { } ``` -### `craneLib.downloadCargoPackage` +### `craneLib.downloadCargoPackages` -`downloadCargoPackage :: set -> drv` +`downloadCargoPackages :: set -> drv` -Download a packaged cargo crate (e.g. from crates.io) and prepare it for -vendoring. +Downloads a collection of cargo crates (e.g. from crates.io) and extract +into a single directory. The registry's `fetchurlExtraArgs` will be passed through to `fetchurl` when downloading the crate, making it possible to influence interacting with the registry's API if necessary. #### Required input attributes -* `checksum`: the (sha256) checksum recorded in the Cargo.lock file -* `name`: the name of the crate -* `source`: the source key recorded in the Cargo.lock file -* `version`: the version of the crate +* `shard`: an identifier for the shard. This will become a part of the derivation name. +* `packages`: a list of attribute sets, each one containing: + * `checksum`: the (sha256) checksum recorded in the Cargo.lock file + * `name`: the name of the crate + * `source`: the source key recorded in the Cargo.lock file + * `version`: the version of the crate ### `craneLib.downloadCargoPackageFromGit` diff --git a/lib/default.nix b/lib/default.nix index e3d891e7..8bb7946a 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -43,7 +43,7 @@ in indexUrl = "https://github.com/rust-lang/crates.io-index"; }; - downloadCargoPackage = callPackage ./downloadCargoPackage.nix { }; + downloadCargoPackages = callPackage ./downloadCargoPackages.nix { }; downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { }; filterCargoSources = callPackage ./filterCargoSources.nix { }; findCargoFiles = callPackage ./findCargoFiles.nix { }; diff --git a/lib/downloadCargoPackage.nix b/lib/downloadCargoPackage.nix deleted file mode 100644 index 137f845c..00000000 --- a/lib/downloadCargoPackage.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ fetchurl -, urlForCargoPackage -, runCommand -}: - -{ name -, version -, checksum -, ... -}@args: -let - pkgInfo = urlForCargoPackage args; - tarball = fetchurl (pkgInfo.fetchurlExtraArgs // { - inherit (pkgInfo) url; - name = "${name}-${version}"; - sha256 = checksum; - }); -in -runCommand "cargo-package-${name}-${version}" { } '' - mkdir -p $out - tar -xzf ${tarball} -C $out --strip-components=1 - echo '{"files":{}, "package":"${checksum}"}' > $out/.cargo-checksum.json -'' diff --git a/lib/downloadCargoPackages.nix b/lib/downloadCargoPackages.nix new file mode 100644 index 00000000..77628e0b --- /dev/null +++ b/lib/downloadCargoPackages.nix @@ -0,0 +1,35 @@ +{ fetchurl +, urlForCargoPackage +, runCommand +, lib +}: + +{shardName, packages}: +let + getTarball = ({ name, version, checksum, ...}@args: let + pkgInfo = urlForCargoPackage args; + in + { + inherit name version checksum; + tarball = fetchurl (pkgInfo.fetchurlExtraArgs // { + inherit (pkgInfo) url; + name = "${name}-${version}"; + sha256 = checksum; + }); + }); + + tarballs = map getTarball packages; + + extract = (tarball: let outPath = "$out/${lib.escapeShellArg "${tarball.name}-${tarball.version}"}"; + in '' + mkdir -p ${outPath} + tar -xf ${tarball.tarball} -C ${outPath} --strip-components=1 + echo '{"files":{}, "package":"${tarball.checksum}"}' > ${outPath}/.cargo-checksum.json + ''); + + name = if builtins.stringLength shardName == 0 then "extract-cargo-packages" else "extract-cargo-packages-${shardName}"; +in +runCommand name { } '' + mkdir -p $out + ${lib.strings.concatMapStrings extract tarballs} +'' diff --git a/lib/vendorCargoRegistries.nix b/lib/vendorCargoRegistries.nix index 4d9e2edc..325081e0 100644 --- a/lib/vendorCargoRegistries.nix +++ b/lib/vendorCargoRegistries.nix @@ -1,4 +1,4 @@ -{ downloadCargoPackage +{ downloadCargoPackages , lib , runCommandLocal }: @@ -44,12 +44,20 @@ let lockPackages; lockedRegistryGroups = groupBy (p: p.source) lockedPackagesFromRegistry; - vendorSingleRegistry = packages: runCommandLocal "vendor-registry" { } '' - mkdir -p $out - ${concatMapStrings (p: '' - ln -s ${escapeShellArg (downloadCargoPackage p)} $out/${escapeShellArg "${p.name}-${p.version}"} - '') packages} - ''; + vendorSingleRegistry = (packages: let + packageCount = builtins.length packages; + shardCharacters = if packageCount < 32 then 0 else if packageCount < 2048 then 1 else 2; + shard = package: builtins.substring 0 shardCharacters (builtins.hashString "sha256" package.name); + grouped = builtins.groupBy shard packages; + vendoredShards = builtins.mapAttrs (shardName: packages: downloadCargoPackages {inherit shardName packages;}) grouped; + vendoredShardsList = builtins.attrValues vendoredShards; + in runCommandLocal "vendor-registry" { } '' + mkdir -p $out + ${concatMapStrings (p: '' + for dir in ${p}/*/; do ln -s "$(realpath "$dir")" "$out/''${dir##*/}"; done + '') vendoredShardsList} + '' + ); parsedCargoConfigTomls = map (p: builtins.fromTOML (readFile p)) cargoConfigs; allCargoRegistries = flatten (map (c: c.registries or [ ]) parsedCargoConfigTomls); From db196fb0eae41dd6aa0faddcad04097495ef8982 Mon Sep 17 00:00:00 2001 From: "james.baker@helsing.ai" Date: Fri, 29 Dec 2023 16:15:07 +0000 Subject: [PATCH 2/3] fmt --- lib/downloadCargoPackages.nix | 16 +++++++++------- lib/vendorCargoRegistries.nix | 18 ++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/downloadCargoPackages.nix b/lib/downloadCargoPackages.nix index 77628e0b..0635db83 100644 --- a/lib/downloadCargoPackages.nix +++ b/lib/downloadCargoPackages.nix @@ -4,10 +4,11 @@ , lib }: -{shardName, packages}: +{ shardName, packages }: let - getTarball = ({ name, version, checksum, ...}@args: let - pkgInfo = urlForCargoPackage args; + getTarball = ({ name, version, checksum, ... }@args: + let + pkgInfo = urlForCargoPackage args; in { inherit name version checksum; @@ -20,11 +21,12 @@ let tarballs = map getTarball packages; - extract = (tarball: let outPath = "$out/${lib.escapeShellArg "${tarball.name}-${tarball.version}"}"; + extract = (tarball: + let outPath = "$out/${lib.escapeShellArg "${tarball.name}-${tarball.version}"}"; in '' - mkdir -p ${outPath} - tar -xf ${tarball.tarball} -C ${outPath} --strip-components=1 - echo '{"files":{}, "package":"${tarball.checksum}"}' > ${outPath}/.cargo-checksum.json + mkdir -p ${outPath} + tar -xf ${tarball.tarball} -C ${outPath} --strip-components=1 + echo '{"files":{}, "package":"${tarball.checksum}"}' > ${outPath}/.cargo-checksum.json ''); name = if builtins.stringLength shardName == 0 then "extract-cargo-packages" else "extract-cargo-packages-${shardName}"; diff --git a/lib/vendorCargoRegistries.nix b/lib/vendorCargoRegistries.nix index 325081e0..b333c7d7 100644 --- a/lib/vendorCargoRegistries.nix +++ b/lib/vendorCargoRegistries.nix @@ -44,14 +44,16 @@ let lockPackages; lockedRegistryGroups = groupBy (p: p.source) lockedPackagesFromRegistry; - vendorSingleRegistry = (packages: let - packageCount = builtins.length packages; - shardCharacters = if packageCount < 32 then 0 else if packageCount < 2048 then 1 else 2; - shard = package: builtins.substring 0 shardCharacters (builtins.hashString "sha256" package.name); - grouped = builtins.groupBy shard packages; - vendoredShards = builtins.mapAttrs (shardName: packages: downloadCargoPackages {inherit shardName packages;}) grouped; - vendoredShardsList = builtins.attrValues vendoredShards; - in runCommandLocal "vendor-registry" { } '' + vendorSingleRegistry = (packages: + let + packageCount = builtins.length packages; + shardCharacters = if packageCount < 32 then 0 else if packageCount < 2048 then 1 else 2; + shard = package: builtins.substring 0 shardCharacters (builtins.hashString "sha256" package.name); + grouped = builtins.groupBy shard packages; + vendoredShards = builtins.mapAttrs (shardName: packages: downloadCargoPackages { inherit shardName packages; }) grouped; + vendoredShardsList = builtins.attrValues vendoredShards; + in + runCommandLocal "vendor-registry" { } '' mkdir -p $out ${concatMapStrings (p: '' for dir in ${p}/*/; do ln -s "$(realpath "$dir")" "$out/''${dir##*/}"; done From efddb2dfd0f13bd95bfe7c34cc7da12252f27767 Mon Sep 17 00:00:00 2001 From: "james.baker@helsing.ai" Date: Fri, 29 Dec 2023 17:12:36 +0000 Subject: [PATCH 3/3] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43fc29e7..a46262c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed * Workspace inheritance of `lints` in git dependencies is now correctly handled +* Crane can now build large (>500 crate) projects inside the OSX sandbox. ## [0.15.1] - 2023-11-30