From 72f462bdbafac035a1633fe064549089c6281ecd Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 20 Nov 2024 05:52:51 +0100 Subject: [PATCH 1/2] pkgs/top-level/release.nix: Don't include non-Hydra attributes with attrNamesOnly The attrNamesOnly feature is used by pkgs/top-level/release-attrpaths-superset.nix to return a superset of all attributes that might be built by Hydra. Before this change it would include all attribute paths to derivations that could be found recursively, ignoring the recurseForDerivations and recurseForRelease attributes that control Hydra's recursion. This had the effect that it would end up with ~266000 attributes, most of which definitely won't be built by Hydra. We can remove those while staying true to the superset notion to end up with just ~97000, a reduction of ~63.6%! This also comes with an eval time reduction from 31.7s to 18.7s (-41.0%)! As an example, all derivations in pypy310Packages don't get included anymore, because it doesn't have a `.recurseForDerivations` set. As a nice side effect, with `--arg enableWarnings false`, no more warnings are printed, because things like `checkpointBuildTools.mkCheckpointedBuild` (which is deprecated) isn't being recursed to anymore. --- pkgs/top-level/release-lib.nix | 23 ++++++++++++++++------- pkgs/top-level/release.nix | 9 +++++---- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index 41221f8b89f3f..c7581c47d3414 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -163,19 +163,26 @@ let (addMetaAttrs { maintainers = crossMaintainers; }); - /* Recursively map a (nested) set of derivations to an isomorphic - set of meta.platforms values. */ - packagePlatforms = mapAttrs (name: value: + /* Recursive for packages and apply a function to them */ + recursiveMapPackages = f: mapAttrs (name: value: if isDerivation value then - value.meta.hydraPlatforms - or (subtractLists (value.meta.badPlatforms or []) - (value.meta.platforms or supportedSystems)) + f value else if value.recurseForDerivations or false || value.recurseForRelease or false then - packagePlatforms value + recursiveMapPackages f value else [] ); + /* Gets the list of Hydra platforms for a derivation */ + getPlatforms = drv: + drv.meta.hydraPlatforms + or (subtractLists (drv.meta.badPlatforms or []) + (drv.meta.platforms or supportedSystems)); + + /* Recursively map a (nested) set of derivations to an isomorphic + set of meta.platforms values. */ + packagePlatforms = recursiveMapPackages getPlatforms; + in { /* Common platform groups on which to test packages. */ inherit (platforms) unix linux darwin cygwin all; @@ -188,6 +195,8 @@ in { lib mapTestOn mapTestOnCross + recursiveMapPackages + getPlatforms packagePlatforms pkgs pkgsFor diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index 3d9de8660282b..a8e65a6a74152 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -321,8 +321,9 @@ let # Conflicts usually cause silent job drops like in # https://github.com/NixOS/nixpkgs/pull/182058 jobs = let - packagePlatforms = if attrNamesOnly then id else release-lib.packagePlatforms; - packageJobs = { + packagePlatforms = release-lib.recursiveMapPackages + (if attrNamesOnly then id else release-lib.getPlatforms); + packageJobs = packagePlatforms pkgs // { haskell.compiler = packagePlatforms pkgs.haskell.compiler; haskellPackages = packagePlatforms pkgs.haskellPackages; # Build selected packages (HLS) for multiple Haskell compilers to rebuild @@ -363,8 +364,8 @@ let }; mapTestOn-packages = if attrNamesOnly - then pkgs // packageJobs - else mapTestOn ((packagePlatforms pkgs) // packageJobs); + then packageJobs + else mapTestOn packageJobs; in unionOfDisjoint nonPackageJobs mapTestOn-packages; From fbbe972898fc25db0dc33272258822c66a9c2870 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 14 Nov 2024 23:14:20 +0100 Subject: [PATCH 2/2] Parallel GH actions workflow for Nixpkgs eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivated by ofborg struggling [1] and its evaluations taking too long, inspired by Jörg's initial PR [2] and Adam's previous attempt to parallelise Nixpkgs evaluation [3], this PR contains initial work to relief ofborg from its evaluation duty by using GitHub Actions to evaluate Nixpkgs. For now this doesn't take care of all of what ofborg does, such as requesting appropriate reviewers or labeling mass rebuilds, but this can be follow-up work. [1]: https://discourse.nixos.org/t/infrastructure-announcement-the-future-of-ofborg-your-help-needed/56025?u=infinisil [2]: https://github.com/NixOS/nixpkgs/pull/352808 [3]: https://github.com/NixOS/nixpkgs/pull/269403 Co-Authored-By: Jörg Thalheim Co-Authored-By: Adam Joseph --- .github/workflows/eval.yml | 139 +++++++++ ci/default.nix | 1 + ci/eval/README.md | 19 ++ ci/eval/default.nix | 273 ++++++++++++++++++ ci/supportedSystems.nix | 6 + lib/tests/release.nix | 16 +- pkgs/test/release/default.nix | 52 ---- pkgs/top-level/release-attrpaths-parallel.nix | 47 +++ pkgs/top-level/release-haskell.nix | 2 +- pkgs/top-level/release-outpaths.nix | 8 +- pkgs/top-level/release.nix | 2 +- 11 files changed, 489 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/eval.yml create mode 100644 ci/eval/README.md create mode 100644 ci/eval/default.nix create mode 100644 ci/supportedSystems.nix delete mode 100644 pkgs/test/release/default.nix create mode 100644 pkgs/top-level/release-attrpaths-parallel.nix diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml new file mode 100644 index 0000000000000..c63496c7255f4 --- /dev/null +++ b/.github/workflows/eval.yml @@ -0,0 +1,139 @@ +name: Eval + +on: pull_request_target + +permissions: + contents: read + +jobs: + attrs: + name: Attributes + runs-on: ubuntu-latest + outputs: + mergedSha: ${{ steps.merged.outputs.mergedSha }} + systems: ${{ steps.systems.outputs.systems }} + steps: + # Important: Because of `pull_request_target`, this doesn't check out the PR, + # but rather the base branch of the PR, which is needed so we don't run untrusted code + - name: Check out the ci directory of the base branch + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: base + sparse-checkout: ci + - name: Check if the PR can be merged and get the test merge commit + id: merged + env: + GH_TOKEN: ${{ github.token }} + run: | + if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then + echo "Checking the merge commit $mergedSha" + echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT" + else + # Skipping so that no notifications are sent + echo "Skipping the rest..." + fi + rm -rf base + - name: Check out the PR at the test merge commit + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # Add this to _all_ subsequent steps to skip them + if: steps.merged.outputs.mergedSha + with: + ref: ${{ env.mergedSha }} + path: nixpkgs + + - name: Install Nix + uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + if: steps.merged.outputs.mergedSha + + - name: Evaluate the list of all attributes and get the systems matrix + id: systems + if: steps.merged.outputs.mergedSha + run: | + nix-build nixpkgs/ci -A eval.attrpathsSuperset + echo "systems=$(> "$GITHUB_OUTPUT" + + - name: Upload the list of all attributes + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + if: steps.merged.outputs.mergedSha + with: + name: paths + path: result/* + + outpaths: + name: Outpaths + runs-on: ubuntu-latest + needs: attrs + # Skip this and future steps if the PR can't be merged + if: needs.attrs.outputs.mergedSha + strategy: + matrix: + system: ${{ fromJSON(needs.attrs.outputs.systems) }} + steps: + - name: Download the list of all attributes + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: paths + path: paths + + - name: Check out the PR at the test merge commit + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ needs.attrs.outputs.mergedSha }} + path: nixpkgs + + - name: Install Nix + uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + + - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes + run: | + nix-build nixpkgs/ci -A eval.singleSystem \ + --argstr evalSystem ${{ matrix.system }} \ + --arg attrpathFile ./paths/paths.json \ + --arg chunkSize 10000 + # If it uses too much memory, slightly decrease chunkSize + + - name: Upload the output paths and eval stats + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + if: needs.attrs.outputs.mergedSha + with: + name: intermediate-${{ matrix.system }} + path: result/* + + process: + name: Process + runs-on: ubuntu-latest + needs: outpaths + steps: + - name: Download output paths and eval stats for all systems + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: intermediate-* + path: intermediate + + - name: Check out the PR at the test merge commit + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ needs.attrs.outputs.mergedSha }} + path: nixpkgs + + - name: Install Nix + uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + + - name: Combine all output paths and eval stats + run: | + nix-build nixpkgs/ci -A eval.combine \ + --arg resultsDir ./intermediate + + - name: Upload the combined results + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: result + path: result/* + + + # TODO: Run this workflow also on `push` (on at least the main development branches) + # Then add an extra step here that waits for the base branch (not the merge base, because that could be very different) + # to have completed the eval, then use + # gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha= + # and follow it to the artifact results, where you can then download the outpaths.json from the base branch + # That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers diff --git a/ci/default.nix b/ci/default.nix index 02b2e948d17b8..f5a6cf7d3dd0e 100644 --- a/ci/default.nix +++ b/ci/default.nix @@ -26,4 +26,5 @@ in inherit pkgs; requestReviews = pkgs.callPackage ./request-reviews { }; codeownersValidator = pkgs.callPackage ./codeowners-validator { }; + eval = pkgs.callPackage ./eval { }; } diff --git a/ci/eval/README.md b/ci/eval/README.md new file mode 100644 index 0000000000000..020e7127f51a9 --- /dev/null +++ b/ci/eval/README.md @@ -0,0 +1,19 @@ +# Nixpkgs CI evaluation + +The code in this directory is used by the [eval.yml](../../.github/workflows/eval.yml) GitHub Actions workflow to evaluate the majority of Nixpkgs for all PRs, effectively making sure that when the development branches are processed by Hydra, no evaluation failures are encountered. + +Furthermore it also allows local evaluation using +``` +nix-build ci -A eval.full \ + --max-jobs 4 + --cores 2 + --arg chunkSize 10000 +``` + +- `--max-jobs`: The maximum number of derivations to run at the same time. Only each [supported system](../supportedSystems.nix) gets a separate derivation, so it doesn't make sense to set this higher than that number. +- `--cores`: The number of cores to use for each job. Recommended to set this to the amount of cores on your system divided by `--max-jobs`. +- `chunkSize`: The number of attributes that are evaluated simultaneously on a single core. Lowering this decreases memory usage at the cost of increased evaluation time. If this is too high, there won't be enough chunks to process them in parallel, and will also increase evaluation time. + +A good default is to set `chunkSize` to 10000, which leads to about 3.6GB max memory usage per core, so suitable for fully utilising machines with 4 cores and 16GB memory, 8 cores and 32GB memory or 16 cores and 64GB memory. + +Note that 16GB memory is the recommended minimum, while with less than 8GB memory evaluation time suffers greatly. diff --git a/ci/eval/default.nix b/ci/eval/default.nix new file mode 100644 index 0000000000000..f213bff611ddb --- /dev/null +++ b/ci/eval/default.nix @@ -0,0 +1,273 @@ +{ + lib, + runCommand, + writeShellScript, + linkFarm, + time, + procps, + nix, + jq, + sta, +}: + +let + nixpkgs = + with lib.fileset; + toSource { + root = ../..; + fileset = unions ( + map (lib.path.append ../..) [ + "default.nix" + "doc" + "lib" + "maintainers" + "nixos" + "pkgs" + ".version" + "ci/supportedSystems.nix" + ] + ); + }; + + supportedSystems = import ../supportedSystems.nix; + + attrpathsSuperset = + runCommand "attrpaths-superset.json" + { + src = nixpkgs; + nativeBuildInputs = [ + nix + time + ]; + env.supportedSystems = builtins.toJSON supportedSystems; + passAsFile = [ "supportedSystems" ]; + } + '' + export NIX_STATE_DIR=$(mktemp -d) + mkdir $out + export GC_INITIAL_HEAP_SIZE=4g + command time -v \ + nix-instantiate --eval --strict --json --show-trace \ + $src/pkgs/top-level/release-attrpaths-superset.nix -A paths \ + --arg enableWarnings false > $out/paths.json + mv "$supportedSystemsPath" $out/systems.json + ''; + + singleSystem = + { + # The system to evaluate. + # Note that this is intentionally not called `system`, + # because `--argstr system` would only be passed to the ci/default.nix file! + evalSystem, + # The path to the `paths.json` file from `attrpathsSuperset` + attrpathFile, + # The number of attributes per chunk, see ./README.md for more info. + chunkSize, + checkMeta ? true, + includeBroken ? true, + # Whether to just evaluate a single chunk for quick testing + quickTest ? false, + }: + let + singleChunk = writeShellScript "single-chunk" '' + set -euo pipefail + chunkSize=$1 + myChunk=$2 + system=$3 + outputDir=$4 + + export NIX_SHOW_STATS=1 + export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk" + echo "Chunk $myChunk on $system start" + set +e + command time -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \ + nix-env -f "${nixpkgs}/pkgs/top-level/release-attrpaths-parallel.nix" \ + --query --available \ + --no-name --attr-path --out-path \ + --show-trace \ + --arg chunkSize "$chunkSize" \ + --arg myChunk "$myChunk" \ + --arg attrpathFile "${attrpathFile}" \ + --arg systems "[ \"$system\" ]" \ + --arg checkMeta ${lib.boolToString checkMeta} \ + --arg includeBroken ${lib.boolToString includeBroken} \ + > "$outputDir/result/$myChunk" + exitCode=$? + set -e + if (( exitCode != 0 )); then + echo "Evaluation failed with exit code $exitCode" + # This immediately halts all xargs processes + kill $PPID + fi + ''; + in + runCommand "nixpkgs-eval-${evalSystem}" + { + nativeBuildInputs = [ + nix + time + procps + jq + ]; + env = { + inherit evalSystem chunkSize; + }; + } + '' + export NIX_STATE_DIR=$(mktemp -d) + nix-store --init + + echo "System: $evalSystem" + cores=$NIX_BUILD_CORES + echo "Cores: $cores" + attrCount=$(jq length "${attrpathFile}") + echo "Attribute count: $attrCount" + echo "Chunk size: $chunkSize" + # Same as `attrCount / chunkSize` but rounded up + chunkCount=$(( (attrCount - 1) / chunkSize + 1 )) + echo "Chunk count: $chunkCount" + + mkdir $out + + # Record and print stats on free memory and swap in the background + ( + while true; do + availMemory=$(free -b | grep Mem | awk '{print $7}') + freeSwap=$(free -b | grep Swap | awk '{print $4}') + echo "Available memory: $(( availMemory / 1024 / 1024 )) MiB, free swap: $(( freeSwap / 1024 / 1024 )) MiB" + + if [[ ! -f "$out/min-avail-memory" ]] || (( availMemory < $(<$out/min-avail-memory) )); then + echo "$availMemory" > $out/min-avail-memory + fi + if [[ ! -f $out/min-free-swap ]] || (( availMemory < $(<$out/min-free-swap) )); then + echo "$freeSwap" > $out/min-free-swap + fi + sleep 4 + done + ) & + + seq_end=$(( chunkCount - 1 )) + + ${lib.optionalString quickTest '' + seq_end=0 + ''} + + chunkOutputDir=$(mktemp -d) + mkdir "$chunkOutputDir"/{result,stats} + + seq -w 0 "$seq_end" | + command time -f "%e" -o "$out/total-time" \ + xargs -I{} -P"$cores" \ + ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir" + + if (( chunkSize * chunkCount != attrCount )); then + # A final incomplete chunk would mess up the stats, don't include it + rm "$chunkOutputDir"/stats/"$seq_end" + fi + + # Make sure the glob doesn't break when there's no files + shopt -s nullglob + cat "$chunkOutputDir"/result/* > $out/paths + cat "$chunkOutputDir"/stats/* > $out/stats.jsonstream + ''; + + combine = + { + resultsDir, + }: + runCommand "combined-result" + { + nativeBuildInputs = [ + jq + sta + ]; + } + '' + mkdir -p $out + + # Transform output paths to JSON + cat ${resultsDir}/*/paths | + jq --sort-keys --raw-input --slurp ' + split("\n") | + map(select(. != "") | split(" ") | map(select(. != ""))) | + map( + { + key: .[0], + value: .[1] | split(";") | map(split("=") | + if length == 1 then + { key: "out", value: .[0] } + else + { key: .[0], value: .[1] } + end) | from_entries} + ) | from_entries + ' > $out/outpaths.json + + # Computes min, mean, error, etc. for a list of values and outputs a JSON from that + statistics() { + local stat=$1 + sta --transpose | + jq --raw-input --argjson stat "$stat" -n ' + [ + inputs | + split("\t") | + { key: .[0], value: (.[1] | fromjson) } + ] | + from_entries | + { + key: ($stat | join(".")), + value: . + }' + } + + # Gets all available number stats (without .sizes because those are constant and not interesting) + readarray -t stats < <(jq -cs '.[0] | del(.sizes) | paths(type == "number")' ${resultsDir}/*/stats.jsonstream) + + # Combines the statistics from all evaluations + { + echo "{ \"key\": \"minAvailMemory\", \"value\": $(cat ${resultsDir}/*/min-avail-memory | sta --brief --min) }" + echo "{ \"key\": \"minFreeSwap\", \"value\": $(cat ${resultsDir}/*/min-free-swap | sta --brief --min) }" + cat ${resultsDir}/*/total-time | statistics '["totalTime"]' + for stat in "''${stats[@]}"; do + cat ${resultsDir}/*/stats.jsonstream | + jq --argjson stat "$stat" 'getpath($stat)' | + statistics "$stat" + done + } | + jq -s from_entries > $out/stats.json + ''; + + full = + { + # Whether to evaluate just a single system, by default all are evaluated + evalSystem ? if quickTest then "x86_64-linux" else null, + # The number of attributes per chunk, see ./README.md for more info. + chunkSize, + quickTest ? false, + }: + let + systems = if evalSystem == null then supportedSystems else [ evalSystem ]; + results = linkFarm "results" ( + map (evalSystem: { + name = evalSystem; + path = singleSystem { + inherit quickTest evalSystem chunkSize; + attrpathFile = attrpathsSuperset + "/paths.json"; + }; + }) systems + ); + in + combine { + resultsDir = results; + }; + +in +{ + inherit + attrpathsSuperset + singleSystem + combine + # The above three are used by separate VMs in a GitHub workflow, + # while the below is intended for testing on a single local machine + full + ; +} diff --git a/ci/supportedSystems.nix b/ci/supportedSystems.nix new file mode 100644 index 0000000000000..471f84b92fc24 --- /dev/null +++ b/ci/supportedSystems.nix @@ -0,0 +1,6 @@ +[ + "aarch64-linux" + "aarch64-darwin" + "x86_64-linux" + "x86_64-darwin" +] diff --git a/lib/tests/release.nix b/lib/tests/release.nix index 084fbd94d34c2..5334498d08449 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -14,19 +14,5 @@ let in pkgs.symlinkJoin { name = "nixpkgs-lib-tests"; - paths = map testWithNix nixVersions ++ - - # - # TEMPORARY MIGRATION MECHANISM - # - # This comment and the expression which follows it should be - # removed as part of resolving this issue: - # - # https://github.com/NixOS/nixpkgs/issues/272591 - # - [(import ../../pkgs/test/release { - inherit pkgs lib nix; - })] - ; - + paths = map testWithNix nixVersions; } diff --git a/pkgs/test/release/default.nix b/pkgs/test/release/default.nix deleted file mode 100644 index 2fdc6d0dc38c5..0000000000000 --- a/pkgs/test/release/default.nix +++ /dev/null @@ -1,52 +0,0 @@ -# Adapted from lib/tests/release.nix -{ pkgs-path ? ../../.. -, pkgs ? import pkgs-path {} -, lib ? pkgs.lib -, nix ? pkgs.nix -}: - -# -# This verifies that release-attrpaths-superset.nix does not encounter -# infinite recursion or non-tryEval-able failures. -# -pkgs.runCommand "all-attrs-eval-under-tryEval" { - nativeBuildInputs = [ - nix - pkgs.gitMinimal - ] ++ lib.optional pkgs.stdenv.hostPlatform.isLinux pkgs.inotify-tools; - strictDeps = true; - - src = with lib.fileset; toSource { - root = pkgs-path; - fileset = unions [ - ../../../default.nix - ../../../doc - ../../../lib - ../../../maintainers - ../../../nixos - ../../../pkgs - ../../../.version - ]; - }; -} -'' - datadir="${nix}/share" - export TEST_ROOT=$(pwd)/test-tmp - export HOME=$(mktemp -d) - export NIX_BUILD_HOOK= - export NIX_CONF_DIR=$TEST_ROOT/etc - export NIX_LOCALSTATE_DIR=$TEST_ROOT/var - export NIX_LOG_DIR=$TEST_ROOT/var/log/nix - export NIX_STATE_DIR=$TEST_ROOT/var/nix - export NIX_STORE_DIR=$TEST_ROOT/store - export PAGER=cat - cacheDir=$TEST_ROOT/binary-cache - - nix-store --init - - echo "Running pkgs/top-level/release-attrpaths-superset.nix" - nix-instantiate --eval --strict --json $src/pkgs/top-level/release-attrpaths-superset.nix -A names > /dev/null - - mkdir $out - echo success > $out/${nix.version} -'' diff --git a/pkgs/top-level/release-attrpaths-parallel.nix b/pkgs/top-level/release-attrpaths-parallel.nix new file mode 100644 index 0000000000000..46d10c6a95041 --- /dev/null +++ b/pkgs/top-level/release-attrpaths-parallel.nix @@ -0,0 +1,47 @@ +# This file works in tandem with ../../ci/eval/default.nix +# It turns ./release-outpaths.nix into chunks of a fixed size +{ + lib ? import ../../lib, + path ? ../.., + # The file containing all available attribute paths, which are split into chunks here + attrpathFile, + chunkSize, + myChunk, + checkMeta, + includeBroken, + systems, +}: + +let + attrpaths = lib.importJSON attrpathFile; + myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths; + + unfiltered = import ./release-outpaths.nix { + inherit path; + inherit checkMeta includeBroken systems; + }; + + # Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths + filtered = + let + recurse = + index: paths: attrs: + lib.mapAttrs ( + name: values: + if attrs ? ${name} then + if lib.any (value: lib.length value <= index + 1) values then + attrs.${name} + else + recurse (index + 1) values attrs.${name} + # Make sure nix-env recurses as well + // { + recurseForDerivations = true; + } + else + null + ) (lib.groupBy (a: lib.elemAt a index) paths); + in + recurse 0 myAttrpaths unfiltered; + +in +filtered diff --git a/pkgs/top-level/release-haskell.nix b/pkgs/top-level/release-haskell.nix index 7a5a87ccf42e4..7df396f4327fc 100644 --- a/pkgs/top-level/release-haskell.nix +++ b/pkgs/top-level/release-haskell.nix @@ -9,7 +9,7 @@ $ hydra-eval-jobs -I . pkgs/top-level/release-haskell.nix */ -{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ] }: +{ supportedSystems ? import ../../ci/supportedSystems.nix }: let diff --git a/pkgs/top-level/release-outpaths.nix b/pkgs/top-level/release-outpaths.nix index 5c433fa542e0c..fd54609efa51b 100644 --- a/pkgs/top-level/release-outpaths.nix +++ b/pkgs/top-level/release-outpaths.nix @@ -12,13 +12,7 @@ , attrNamesOnly ? false # Set this to `null` to build for builtins.currentSystem only -, systems ? [ - "aarch64-linux" - "aarch64-darwin" - #"i686-linux" # !!! - "x86_64-linux" - "x86_64-darwin" - ] +, systems ? import ../../ci/supportedSystems.nix }: let lib = import (path + "/lib"); diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index a8e65a6a74152..838ac6ec3934e 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -12,7 +12,7 @@ , system ? builtins.currentSystem , officialRelease ? false # The platform doubles for which we build Nixpkgs. -, supportedSystems ? [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ] +, supportedSystems ? import ../../ci/supportedSystems.nix # The platform triples for which we build bootstrap tools. , bootstrapConfigs ? [ "aarch64-apple-darwin"