diff --git a/docs/API.md b/docs/API.md index bb5cb01c..1729c443 100644 --- a/docs/API.md +++ b/docs/API.md @@ -859,22 +859,24 @@ craneLib.devShell { } ``` -### `craneLib.downloadCargoPackage` +### `craneLib.downloadCargoPackages` -`downloadCargoPackage :: set -> drv` +`downloadCargoPackages :: set -> drv` -Download a packaged cargo crate (e.g. from crates.io) and prepare it for -vendoring. +Downloads a collection of cargo crates (e.g. from crates.io) and extract +into a single directory. The registry's `fetchurlExtraArgs` will be passed through to `fetchurl` when downloading the crate, making it possible to influence interacting with the registry's API if necessary. #### Required input attributes -* `checksum`: the (sha256) checksum recorded in the Cargo.lock file -* `name`: the name of the crate -* `source`: the source key recorded in the Cargo.lock file -* `version`: the version of the crate +* `shard`: an identifier for the shard. This will become a part of the derivation name. +* `packages`: a list of attribute sets, each one containing: + * `checksum`: the (sha256) checksum recorded in the Cargo.lock file + * `name`: the name of the crate + * `source`: the source key recorded in the Cargo.lock file + * `version`: the version of the crate ### `craneLib.downloadCargoPackageFromGit` diff --git a/lib/default.nix b/lib/default.nix index e3d891e7..8bb7946a 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -43,7 +43,7 @@ in indexUrl = "https://github.com/rust-lang/crates.io-index"; }; - downloadCargoPackage = callPackage ./downloadCargoPackage.nix { }; + downloadCargoPackages = callPackage ./downloadCargoPackages.nix { }; downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { }; filterCargoSources = callPackage ./filterCargoSources.nix { }; findCargoFiles = callPackage ./findCargoFiles.nix { }; diff --git a/lib/downloadCargoPackage.nix b/lib/downloadCargoPackage.nix deleted file mode 100644 index 137f845c..00000000 --- a/lib/downloadCargoPackage.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ fetchurl -, urlForCargoPackage -, runCommand -}: - -{ name -, version -, checksum -, ... -}@args: -let - pkgInfo = urlForCargoPackage args; - tarball = fetchurl (pkgInfo.fetchurlExtraArgs // { - inherit (pkgInfo) url; - name = "${name}-${version}"; - sha256 = checksum; - }); -in -runCommand "cargo-package-${name}-${version}" { } '' - mkdir -p $out - tar -xzf ${tarball} -C $out --strip-components=1 - echo '{"files":{}, "package":"${checksum}"}' > $out/.cargo-checksum.json -'' diff --git a/lib/downloadCargoPackages.nix b/lib/downloadCargoPackages.nix new file mode 100644 index 00000000..77628e0b --- /dev/null +++ b/lib/downloadCargoPackages.nix @@ -0,0 +1,35 @@ +{ fetchurl +, urlForCargoPackage +, runCommand +, lib +}: + +{shardName, packages}: +let + getTarball = ({ name, version, checksum, ...}@args: let + pkgInfo = urlForCargoPackage args; + in + { + inherit name version checksum; + tarball = fetchurl (pkgInfo.fetchurlExtraArgs // { + inherit (pkgInfo) url; + name = "${name}-${version}"; + sha256 = checksum; + }); + }); + + tarballs = map getTarball packages; + + extract = (tarball: let outPath = "$out/${lib.escapeShellArg "${tarball.name}-${tarball.version}"}"; + in '' + mkdir -p ${outPath} + tar -xf ${tarball.tarball} -C ${outPath} --strip-components=1 + echo '{"files":{}, "package":"${tarball.checksum}"}' > ${outPath}/.cargo-checksum.json + ''); + + name = if builtins.stringLength shardName == 0 then "extract-cargo-packages" else "extract-cargo-packages-${shardName}"; +in +runCommand name { } '' + mkdir -p $out + ${lib.strings.concatMapStrings extract tarballs} +'' diff --git a/lib/vendorCargoRegistries.nix b/lib/vendorCargoRegistries.nix index 4d9e2edc..325081e0 100644 --- a/lib/vendorCargoRegistries.nix +++ b/lib/vendorCargoRegistries.nix @@ -1,4 +1,4 @@ -{ downloadCargoPackage +{ downloadCargoPackages , lib , runCommandLocal }: @@ -44,12 +44,20 @@ let lockPackages; lockedRegistryGroups = groupBy (p: p.source) lockedPackagesFromRegistry; - vendorSingleRegistry = packages: runCommandLocal "vendor-registry" { } '' - mkdir -p $out - ${concatMapStrings (p: '' - ln -s ${escapeShellArg (downloadCargoPackage p)} $out/${escapeShellArg "${p.name}-${p.version}"} - '') packages} - ''; + vendorSingleRegistry = (packages: let + packageCount = builtins.length packages; + shardCharacters = if packageCount < 32 then 0 else if packageCount < 2048 then 1 else 2; + shard = package: builtins.substring 0 shardCharacters (builtins.hashString "sha256" package.name); + grouped = builtins.groupBy shard packages; + vendoredShards = builtins.mapAttrs (shardName: packages: downloadCargoPackages {inherit shardName packages;}) grouped; + vendoredShardsList = builtins.attrValues vendoredShards; + in runCommandLocal "vendor-registry" { } '' + mkdir -p $out + ${concatMapStrings (p: '' + for dir in ${p}/*/; do ln -s "$(realpath "$dir")" "$out/''${dir##*/}"; done + '') vendoredShardsList} + '' + ); parsedCargoConfigTomls = map (p: builtins.fromTOML (readFile p)) cargoConfigs; allCargoRegistries = flatten (map (c: c.registries or [ ]) parsedCargoConfigTomls);