diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 62facba96b72e..a7c3a94a67880 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -40,14 +40,6 @@ jobs: - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 if: env.mergedSha - - name: Enable swap - if: env.mergedSha - run: | - sudo fallocate -l 10G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - - name: Check eval if: env.mergedSha run: ./ci/eval-nixpkgs.sh --system "${{ matrix.system }}" diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index d62ef07cfa5e9..ca1a22a99ac87 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -1,25 +1,37 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p moreutils -I nixpkgs=channel:nixpkgs-unstable +#!nix-shell -i bash -p coreutils moreutils -I nixpkgs=channel:nixpkgs-unstable set -euxo pipefail -system="x86_64-linux" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" NIXPKGS_PATH="$(readlink -f "$SCRIPT_DIR"/..)" +system="x86_64-linux" +quick_test=0 +CORES=$(nproc) + parseArgs() { - while [[ $# -gt 0 ]]; do - case $1 in - --system) - system=$2 - shift 2 - ;; - *) - echo "Unknown argument: $1" - exit 1 - ;; - esac - done + while [[ $# -gt 0 ]]; do + arg=$1 + shift + case "$arg" in + --system) + system=$1 + shift 1 + ;; + --cores) + CORES=$1 + shift 1 + ;; + --quick-test) + quick_test=1 + ;; + *) + echo "Unknown argument: $arg" + exit 1 + ;; + esac + done } main() { @@ -27,34 +39,50 @@ main() { tmpdir=$(mktemp -d) trap 'rm -rf "$tmpdir"' EXIT - nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths > "$tmpdir/paths.json" + nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths >"$tmpdir/paths.json" - CORES=$(nproc) # Originally @amjoseph: note that the number of processes spawned is four times # the number of cores -- this helps in two ways: # 1. Keeping cores busy while I/O operations are in flight # 2. Since the amount of time needed for the jobs is *not* balanced # this minimizes the "tail latency" for the very last job to finish # (on one core) by making the job size smaller. - NUM_CHUNKS=$(( 4 * CORES )) - + local num_chunks=$((4 * CORES)) + local seq_end=$((num_chunks - 1)) + if [[ $quick_test -eq 1 ]]; then + seq_end=0 + fi ( - set +e - parallel -j "$CORES" \ - nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ - --arg systems "[\"$system\"]" \ - -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \ - --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk \ - -- $(seq 0 $(( NUM_CHUNKS - 1 ))) > "$tmpdir/paths" - echo $? > "$tmpdir/exit-code" + set +e + seq 0 $seq_end | xargs -P "$CORES" -I {} nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ + --arg systems "[\"$system\"]" \ + -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \ + --arg numChunks "$num_chunks" --show-trace --arg myChunk {} >"$tmpdir/paths" + echo $? >"$tmpdir/exit-code" ) & pid=$! while kill -0 "$pid"; do free -g >&2 sleep 20 done - jq --raw-input --slurp 'split("\n") | map(select(. != "") | split(" ") | map(select(. != "")) | { key: .[0], value: .[1] }) | from_entries' "$tmpdir/paths" + + # Extract derivation outputs for each attribute into a JSON object + # i.e. { "attr1": { "out": "/nix/store/...", "dev": "/nix/store/..." }, ... } + jq --raw-input --slurp ' + split("\n") | + map(select(. != "") | split(" ") | map(select(. != ""))) | + map( + { + key: .[0], + value: .[1] | split(";") | map(split("=") | + if length == 1 then + { key: "out", value: .[0] } + else + { key: .[0], value: .[1] } + end) | from_entries} + ) | from_entries + ' "$tmpdir/paths" exit "$(cat "$tmpdir/exit-code")" } diff --git a/ci/parallel.nix b/ci/parallel.nix index 8c4a4f3928c25..074c545fe1170 100644 --- a/ci/parallel.nix +++ b/ci/parallel.nix @@ -1,26 +1,25 @@ /* -Invocation: + Invocation: + Invocation; note that the number of processes spawned is four times + the number of cores -- this helps in two ways: -Invocation; note that the number of processes spawned is four times -the number of cores -- this helps in two ways: - -1. Keeping cores busy while I/O operations are in flight - -2. Since the amount of time needed for the jobs is *not* balanced - this minimizes the "tail latency" for the very last job to finish - (on one core) by making the job size smaller. + 1. Keeping cores busy while I/O operations are in flight + 2. Since the amount of time needed for the jobs is *not* balanced + this minimizes the "tail latency" for the very last job to finish + (on one core) by making the job size smaller. */ # see pkgs/top-level/nohydra -{ lib ? import ../lib -, checkMeta -, includeBroken ? true -, path ? ./.. -, systems -, myChunk -, numChunks -, attrPathFile +{ + lib ? import ../lib, + checkMeta, + includeBroken ? true, + path ? ./.., + systems, + myChunk, + numChunks, + attrPathFile, }: let @@ -28,34 +27,37 @@ let chunkSize = (lib.length attrPaths) / numChunks; myPaths = let - dropped = lib.drop (chunkSize*myChunk) attrPaths; + dropped = lib.drop (chunkSize * myChunk) attrPaths; in - if myChunk == numChunks - 1 - then dropped - else lib.take chunkSize dropped; + if myChunk == numChunks - 1 then dropped else lib.take chunkSize dropped; unfiltered = import ../pkgs/top-level/release-outpaths.nix { - inherit checkMeta path includeBroken systems; + inherit + checkMeta + path + includeBroken + systems + ; }; - f = i: m: a: - lib.mapAttrs (name: values: + f = + i: m: a: + lib.mapAttrs ( + name: values: if a ? ${name} then - if lib.any (value: lib.length value <= i + 1) values then - a.${name} - else - f (i + 1) values a.${name} + if lib.any (value: lib.length value <= i + 1) values then a.${name} else f (i + 1) values a.${name} else null ) (lib.groupBy (a: lib.elemAt a i) m); filtered = f 0 myPaths unfiltered; - recurseEverywhere = val: - if lib.isDerivation val || !(lib.isAttrs val) - then val - else (builtins.mapAttrs (_: v: recurseEverywhere v) val) - // { recurseForDerivations = true; }; + recurseEverywhere = + val: + if lib.isDerivation val || !(lib.isAttrs val) then + val + else + (builtins.mapAttrs (_: v: recurseEverywhere v) val) // { recurseForDerivations = true; }; in - recurseEverywhere filtered +recurseEverywhere filtered