From f2d312a4e4e17cbdca2743391b5d13d88cdb91c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:11:33 +0100 Subject: [PATCH 1/8] ci/parallel.nix: nixfmt --- ci/parallel.nix | 70 +++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/ci/parallel.nix b/ci/parallel.nix index 8c4a4f3928c25..074c545fe1170 100644 --- a/ci/parallel.nix +++ b/ci/parallel.nix @@ -1,26 +1,25 @@ /* -Invocation: + Invocation: + Invocation; note that the number of processes spawned is four times + the number of cores -- this helps in two ways: -Invocation; note that the number of processes spawned is four times -the number of cores -- this helps in two ways: - -1. Keeping cores busy while I/O operations are in flight - -2. Since the amount of time needed for the jobs is *not* balanced - this minimizes the "tail latency" for the very last job to finish - (on one core) by making the job size smaller. + 1. Keeping cores busy while I/O operations are in flight + 2. Since the amount of time needed for the jobs is *not* balanced + this minimizes the "tail latency" for the very last job to finish + (on one core) by making the job size smaller. */ # see pkgs/top-level/nohydra -{ lib ? import ../lib -, checkMeta -, includeBroken ? true -, path ? ./.. -, systems -, myChunk -, numChunks -, attrPathFile +{ + lib ? import ../lib, + checkMeta, + includeBroken ? true, + path ? ./.., + systems, + myChunk, + numChunks, + attrPathFile, }: let @@ -28,34 +27,37 @@ let chunkSize = (lib.length attrPaths) / numChunks; myPaths = let - dropped = lib.drop (chunkSize*myChunk) attrPaths; + dropped = lib.drop (chunkSize * myChunk) attrPaths; in - if myChunk == numChunks - 1 - then dropped - else lib.take chunkSize dropped; + if myChunk == numChunks - 1 then dropped else lib.take chunkSize dropped; unfiltered = import ../pkgs/top-level/release-outpaths.nix { - inherit checkMeta path includeBroken systems; + inherit + checkMeta + path + includeBroken + systems + ; }; - f = i: m: a: - lib.mapAttrs (name: values: + f = + i: m: a: + lib.mapAttrs ( + name: values: if a ? ${name} then - if lib.any (value: lib.length value <= i + 1) values then - a.${name} - else - f (i + 1) values a.${name} + if lib.any (value: lib.length value <= i + 1) values then a.${name} else f (i + 1) values a.${name} else null ) (lib.groupBy (a: lib.elemAt a i) m); filtered = f 0 myPaths unfiltered; - recurseEverywhere = val: - if lib.isDerivation val || !(lib.isAttrs val) - then val - else (builtins.mapAttrs (_: v: recurseEverywhere v) val) - // { recurseForDerivations = true; }; + recurseEverywhere = + val: + if lib.isDerivation val || !(lib.isAttrs val) then + val + else + (builtins.mapAttrs (_: v: recurseEverywhere v) val) // { recurseForDerivations = true; }; in - recurseEverywhere filtered +recurseEverywhere filtered From 1074648def45b873d6b5e94d586a304a67d1e449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:22:07 +0100 Subject: [PATCH 2/8] ci/eval: drop swap Turns out we don't actually need it --- .github/workflows/eval.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 62facba96b72e..a7c3a94a67880 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -40,14 +40,6 @@ jobs: - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 if: env.mergedSha - - name: Enable swap - if: env.mergedSha - run: | - sudo fallocate -l 10G /swapfile - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - - name: Check eval if: env.mergedSha run: ./ci/eval-nixpkgs.sh --system "${{ matrix.system }}" From 7d887a5c04b998d5e3fb1b40d9efb228aff1d06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:24:35 +0100 Subject: [PATCH 3/8] ci/eval: reformat with shfmt the file was using mixed indentation (2 vs 4 spaces) --- ci/eval-nixpkgs.sh | 47 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index d62ef07cfa5e9..c53e30660311e 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -4,22 +4,22 @@ set -euxo pipefail system="x86_64-linux" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" NIXPKGS_PATH="$(readlink -f "$SCRIPT_DIR"/..)" parseArgs() { - while [[ $# -gt 0 ]]; do - case $1 in - --system) - system=$2 - shift 2 - ;; - *) - echo "Unknown argument: $1" - exit 1 - ;; - esac - done + while [[ $# -gt 0 ]]; do + case $1 in + --system) + system=$2 + shift 2 + ;; + *) + echo "Unknown argument: $1" + exit 1 + ;; + esac + done } main() { @@ -27,7 +27,7 @@ main() { tmpdir=$(mktemp -d) trap 'rm -rf "$tmpdir"' EXIT - nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths > "$tmpdir/paths.json" + nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths >"$tmpdir/paths.json" CORES=$(nproc) # Originally @amjoseph: note that the number of processes spawned is four times @@ -36,18 +36,17 @@ main() { # 2. Since the amount of time needed for the jobs is *not* balanced # this minimizes the "tail latency" for the very last job to finish # (on one core) by making the job size smaller. - NUM_CHUNKS=$(( 4 * CORES )) - + NUM_CHUNKS=$((4 * CORES)) ( - set +e - parallel -j "$CORES" \ - nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ - --arg systems "[\"$system\"]" \ - -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \ - --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk \ - -- $(seq 0 $(( NUM_CHUNKS - 1 ))) > "$tmpdir/paths" - echo $? > "$tmpdir/exit-code" + set +e + parallel -j "$CORES" \ + nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ + --arg systems "[\"$system\"]" \ + -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \ + --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk \ + -- $(seq 0 $((NUM_CHUNKS - 1))) >"$tmpdir/paths" + echo $? >"$tmpdir/exit-code" ) & pid=$! while kill -0 "$pid"; do From 799336430f694ff94bd345e82afc4d82b1d0a0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:31:04 +0100 Subject: [PATCH 4/8] ci/eval: depend on coreutils nproc is only part of coreutils and not present on macOS --- ci/eval-nixpkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index c53e30660311e..08d215558e155 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p moreutils -I nixpkgs=channel:nixpkgs-unstable +#!nix-shell -i bash -p coreutils moreutils -I nixpkgs=channel:nixpkgs-unstable set -euxo pipefail From c5ec387ea384b0d1217ef329754c48528ee81e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:36:32 +0100 Subject: [PATCH 5/8] ci/eval: allow to override cores this is useful to control how much RAM is used during evaluation --- ci/eval-nixpkgs.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index 08d215558e155..1a8312d026f67 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -4,6 +4,7 @@ set -euxo pipefail system="x86_64-linux" +CORES=$(nproc) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" NIXPKGS_PATH="$(readlink -f "$SCRIPT_DIR"/..)" @@ -14,6 +15,10 @@ parseArgs() { system=$2 shift 2 ;; + --cores) + CORES=$2 + shift 2 + ;; *) echo "Unknown argument: $1" exit 1 @@ -29,7 +34,6 @@ main() { nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths >"$tmpdir/paths.json" - CORES=$(nproc) # Originally @amjoseph: note that the number of processes spawned is four times # the number of cores -- this helps in two ways: # 1. Keeping cores busy while I/O operations are in flight From 000b0679e94038e12dc64732c2e8f973a15994e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:39:03 +0100 Subject: [PATCH 6/8] ci/eval: use xargs instead of parallel saves us from having to install perl for this script. --- ci/eval-nixpkgs.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index 1a8312d026f67..a8cd3bc524a9b 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -44,12 +44,10 @@ main() { ( set +e - parallel -j "$CORES" \ - nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ + seq 0 $((NUM_CHUNKS - 1)) | xargs -P "$CORES" -I {} nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ --arg systems "[\"$system\"]" \ -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \ - --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk \ - -- $(seq 0 $((NUM_CHUNKS - 1))) >"$tmpdir/paths" + --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk {} >"$tmpdir/paths" echo $? >"$tmpdir/exit-code" ) & pid=$! From 8a5ee19c197c33dd7bd9de32f7e3239fa84c53e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 08:54:38 +0100 Subject: [PATCH 7/8] ci/eval: add quick test flag --- ci/eval-nixpkgs.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index a8cd3bc524a9b..377c0c8cbdcdb 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -3,24 +3,31 @@ set -euxo pipefail -system="x86_64-linux" -CORES=$(nproc) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" NIXPKGS_PATH="$(readlink -f "$SCRIPT_DIR"/..)" +system="x86_64-linux" +quick_test=0 +CORES=$(nproc) + parseArgs() { while [[ $# -gt 0 ]]; do - case $1 in + arg=$1 + shift + case "$arg" in --system) - system=$2 - shift 2 + system=$1 + shift 1 ;; --cores) - CORES=$2 - shift 2 + CORES=$1 + shift 1 + ;; + --quick-test) + quick_test=1 ;; *) - echo "Unknown argument: $1" + echo "Unknown argument: $arg" exit 1 ;; esac @@ -40,14 +47,18 @@ main() { # 2. Since the amount of time needed for the jobs is *not* balanced # this minimizes the "tail latency" for the very last job to finish # (on one core) by making the job size smaller. - NUM_CHUNKS=$((4 * CORES)) + local num_chunks=$((4 * CORES)) + local seq_end=$((num_chunks - 1)) + if [[ $quick_test -eq 1 ]]; then + seq_end=0 + fi ( set +e - seq 0 $((NUM_CHUNKS - 1)) | xargs -P "$CORES" -I {} nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ + seq 0 $seq_end | xargs -P "$CORES" -I {} nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \ --arg systems "[\"$system\"]" \ -f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \ - --arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk {} >"$tmpdir/paths" + --arg numChunks "$num_chunks" --show-trace --arg myChunk {} >"$tmpdir/paths" echo $? >"$tmpdir/exit-code" ) & pid=$! From af2417f9d90afb1fb539b12e478048631e9641f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Nov 2024 09:39:07 +0100 Subject: [PATCH 8/8] ci/eval: seperate different outputs in json --- ci/eval-nixpkgs.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ci/eval-nixpkgs.sh b/ci/eval-nixpkgs.sh index 377c0c8cbdcdb..ca1a22a99ac87 100755 --- a/ci/eval-nixpkgs.sh +++ b/ci/eval-nixpkgs.sh @@ -66,7 +66,23 @@ main() { free -g >&2 sleep 20 done - jq --raw-input --slurp 'split("\n") | map(select(. != "") | split(" ") | map(select(. != "")) | { key: .[0], value: .[1] }) | from_entries' "$tmpdir/paths" + + # Extract derivation outputs for each attribute into a JSON object + # i.e. { "attr1": { "out": "/nix/store/...", "dev": "/nix/store/..." }, ... } + jq --raw-input --slurp ' + split("\n") | + map(select(. != "") | split(" ") | map(select(. != ""))) | + map( + { + key: .[0], + value: .[1] | split(";") | map(split("=") | + if length == 1 then + { key: "out", value: .[0] } + else + { key: .[0], value: .[1] } + end) | from_entries} + ) | from_entries + ' "$tmpdir/paths" exit "$(cat "$tmpdir/exit-code")" }