forked from NixOS/nixpkgs
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
11 changed files
with
317 additions
and
186 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,213 @@ | ||
| { | ||
| lib, | ||
| runCommand, | ||
| writeShellScript, | ||
| time, | ||
| procps, | ||
| nix, | ||
| jq, | ||
| }: | ||
|
|
||
| # Use the GitHub Actions cache to cache /nix/store | ||
| # Although can't really merge.. | ||
| # Use artifacts and pass files manually, also doesn't have to repeat eval then | ||
| let | ||
| nixpkgs = with lib.fileset; toSource { | ||
| root = ../..; | ||
| fileset = unions (map (lib.path.append ../..) [ | ||
| "default.nix" | ||
| "doc" | ||
| "lib" | ||
| "maintainers" | ||
| "nixos" | ||
| "ci" | ||
| "pkgs" | ||
| ".version" | ||
| ]); | ||
| }; | ||
|
|
||
| attrpathsSuperset = | ||
| runCommand "attrpaths-superset.json" { | ||
| src = nixpkgs; | ||
| nativeBuildInputs = [ | ||
| nix | ||
| ]; | ||
| } | ||
| '' | ||
| export NIX_STATE_DIR=$(mktemp -d) | ||
| mkdir $out | ||
| nix-instantiate --eval --strict --json $src/pkgs/top-level/release-attrpaths-superset.nix -A paths > $out/paths.json | ||
| ''; | ||
|
|
||
| # Takes a path to an attrpathsSuperset result and computes the result of evaluating it entirely | ||
| evalPlan = | ||
| { | ||
| # null means as many cores as your machine has | ||
| cores ? null, | ||
| # How many attributes to be evaluating at any single time. | ||
| # This effectively limits the maximum memory usage. | ||
| # Decrease this if too much memory is used | ||
| simultaneousAttrsPerSystem ? 100000, | ||
| checkMeta ? true, | ||
| includeBroken ? true, | ||
| # TODO | ||
| quickTest ? false, | ||
| }: | ||
| runCommand "eval-plan" { | ||
| nativeBuildInputs = [ | ||
| jq | ||
| ]; | ||
| env.cores = toString cores; | ||
| supportedSystems = builtins.toJSON (import ../supportedSystems.nix); | ||
| passAsFile = [ "supportedSystems" ]; | ||
| } '' | ||
| if [[ -z "$cores" ]]; then | ||
| cores=$(nproc) | ||
| fi | ||
| echo "Cores: $cores" | ||
| num_attrs=$(jq length "${attrpathsSuperset}/paths.json") | ||
| echo "Attribute count: $num_attrs" | ||
| chunk_size=$(( ${toString simultaneousAttrsPerSystem} / cores )) | ||
| echo "Chunk size: $chunk_size" | ||
| # Same as `num_attrs / chunk_size` but rounded up | ||
| num_chunks=$(( (num_attrs - 1) / chunk_size + 1 )) | ||
| echo "Chunk count: $num_chunks" | ||
| mkdir -p $out/systems | ||
| mv "$supportedSystemsPath" $out/systems.json | ||
| echo "Systems: $(<$out/systems.json)" | ||
| for system in $(jq -r '.[]' "$out/systems.json"); do | ||
| mkdir -p "$out/systems/$system/chunks" | ||
| printf "%s" "$cores" > "$out/systems/$system/cores" | ||
| for chunk in $(seq -w 0 "$(( num_chunks - 1 ))"); do | ||
| jq '{ | ||
| paths: .[($chunk * $chunk_size):(($chunk + 1) * $chunk_size)], | ||
| systems: [ $system ], | ||
| checkMeta: $checkMeta, | ||
| includeBroken: $includeBroken | ||
| }' \ | ||
| --argjson chunk "$chunk" \ | ||
| --argjson chunk_size "$chunk_size" \ | ||
| --arg system "$system" \ | ||
| --argjson checkMeta "${lib.boolToString checkMeta}" \ | ||
| --argjson includeBroken "${lib.boolToString includeBroken}" \ | ||
| ${attrpathsSuperset}/paths.json \ | ||
| > "$out/systems/$system/chunks/$chunk.json" | ||
| done | ||
| done | ||
| ''; | ||
|
|
||
|
|
||
| singleSystem = | ||
| let | ||
| singleChunk = writeShellScript "chunk" '' | ||
| set -euo pipefail | ||
| chunkFile=$1 | ||
| outputDir=$2 | ||
| nix-env -f "${nixpkgs}/ci/eval/parallel.nix" \ | ||
| --query --available \ | ||
| --no-name --attr-path --out-path \ | ||
| --show-trace \ | ||
| --arg chunkFile "$chunkFile" > "$outputDir/$(basename "$chunkFile")" | ||
| ''; | ||
| in | ||
| { | ||
| systemDir, | ||
| }: | ||
| runCommand "nixpkgs-eval-system-${baseNameOf systemDir}" { | ||
| nativeBuildInputs = [ | ||
| nix | ||
| time | ||
| procps | ||
| ]; | ||
| } '' | ||
| export NIX_STATE_DIR=$(mktemp -d) | ||
| nix-store --init | ||
| cores=$(<${systemDir}/cores) | ||
| chunkOutputDir=$(mktemp -d) | ||
| ( | ||
| while true; do | ||
| free -g | ||
| sleep 20 | ||
| done | ||
| ) & | ||
| find ${systemDir}/chunks -name '*.json' -print0 | | ||
| command time -v xargs -0 -t -I{} -P"$cores" \ | ||
| ${singleChunk} {} "$chunkOutputDir" | ||
| cat "$chunkOutputDir"/* > $out/paths | ||
| ''; | ||
|
|
||
| combine = | ||
| { | ||
|
|
||
| }: null; | ||
|
|
||
| # mkdir $out | ||
| # for system in ${lib.escapeShellArgs (import ../supportedSystems.nix)}; do | ||
|
|
||
| # done | ||
|
|
||
| #''; | ||
|
|
||
| # Files of the form | ||
| # $out/<matrix-index>/<number>.json | ||
| # { | ||
| # paths: [ [ ... ], ... ], | ||
| # system: ..., | ||
| # } | ||
|
|
||
| #attrsSuperset = import ../../pkgs/test/release { | ||
| # # Important: This is used to get dependencies for the evaluation itself. | ||
| # # Because we don't want it to take forever on staging PRs, | ||
| # # this uses the pinned CI version instead. | ||
| # inherit pkgs; | ||
| #}; | ||
|
|
||
| # How to get the list of supported systems? | ||
| # ci/supportedSystems.nix | ||
|
|
||
| #chunkResult = | ||
| # { | ||
| # # null means all default systems | ||
| # # Pass a list of strings to select specific ones | ||
| # system, | ||
| # attrsSupersetFile, | ||
| # myChunk, | ||
| # }: | ||
| # pkgs.runCommand "chunk-result-${system}-${toString myChunk}" '' | ||
| # ''; | ||
|
|
||
| #combinedResult = | ||
| # { | ||
| # chunkResults, | ||
| # }: | ||
| # pkgs.runCommand "combined-result" { | ||
| # passAsFile = [ "jqScript" ]; | ||
| # jqScript = /* jq */ '' | ||
| # split("\n") | | ||
| # map(select(. != "") | split(" ") | map(select(. != ""))) | | ||
| # map( | ||
| # { | ||
| # key: .[0], | ||
| # value: .[1] | split(";") | map(split("=") | | ||
| # if length == 1 then | ||
| # { key: "out", value: .[0] } | ||
| # else | ||
| # { key: .[0], value: .[1] } | ||
| # end) | from_entries} | ||
| # ) | from_entries | ||
| # ''; | ||
| # } '' | ||
| # cat ${lib.escapeShellArgs chunkResults} | | ||
| # jq --raw-input --slurp -f "$jqScriptPath" "$tmpdir/paths" \ | ||
| # > $out/outpaths.json | ||
| # ''; | ||
| in | ||
| { | ||
| inherit attrpathsSuperset evalPlan singleSystem; | ||
| } |
Oops, something went wrong.