From db6b843d7b9efc3b4581960baa5e456f5dfe5db0 Mon Sep 17 00:00:00 2001 From: Dominik Kapusta Date: Fri, 22 Nov 2024 21:52:27 +0100 Subject: [PATCH] Split variants creation into 2 batches --- .../asana-get-build-variants-list/action.yml | 9 ++- .../get_build_variants_list.sh | 64 +++++++++++++------ .github/workflows/create_variant.yml | 2 + .github/workflows/create_variants.yml | 34 ++++++++-- 4 files changed, 82 insertions(+), 27 deletions(-) diff --git a/.github/actions/asana-get-build-variants-list/action.yml b/.github/actions/asana-get-build-variants-list/action.yml index 38543f750a..fc112d5e70 100644 --- a/.github/actions/asana-get-build-variants-list/action.yml +++ b/.github/actions/asana-get-build-variants-list/action.yml @@ -19,9 +19,12 @@ inputs: required: true type: string outputs: - build-variants: - description: "The list of build variants to create" - value: ${{ steps.get-build-variants-task.outputs.build-variants }} + build-variants-1: + description: "The list of build variants to create (batch 1)" + value: ${{ steps.get-build-variants-task.outputs.build-variants-1 }} + build-variants-2: + description: "The list of build variants to create (batch 2)" + value: ${{ steps.get-build-variants-task.outputs.build-variants-2 }} runs: using: "composite" steps: diff --git a/.github/actions/asana-get-build-variants-list/get_build_variants_list.sh b/.github/actions/asana-get-build-variants-list/get_build_variants_list.sh index d9308dc0d4..de58ebefe4 100755 --- a/.github/actions/asana-get-build-variants-list/get_build_variants_list.sh +++ b/.github/actions/asana-get-build-variants-list/get_build_variants_list.sh @@ -22,9 +22,7 @@ _create_origins_and_variants() { jq -c '.data[] | select(.custom_fields[] | select(.name == "'"${origin_field}"'").text_value != null) | {origin: (.custom_fields[] | select(.name == "'"${origin_field}"'") | .text_value), variant: (.custom_fields[] | select(.name == "'"${atb_field}"'") | .text_value)} - | if .variant != null then {origin}, {origin, variant} else {origin} end' <<< "$response" \ - | tr '\n' ',' \ - | sed 's/,$//' + | if .variant != null then {origin}, {origin, variant} else {origin} end' <<< "$response" } # Fetch all the Asana tasks in the section specified by ORIGIN_ASANA_SECTION_ID for a project. @@ -56,7 +54,7 @@ _fetch_origin_tasks() { fi done - echo "${origin_variants[*]}" | tr ' ' ',' + printf "%s\n" "${origin_variants[@]}" } # Create a JSON string from the list of ATB items passed. @@ -70,9 +68,7 @@ _create_atb_variant_pairs() { # remove the trailing comma at the end of the line. jq -R -c 'split(",") | map({variant: .}) - | .[]' <<< "$response" \ - | tr '\n' ',' \ - | sed 's/,$//' + | .[]' <<< "$response" } # Fetches all the ATB variants defined in the ATB_ASANA_TASK_ID at the Variants list (comma separated) section. @@ -92,21 +88,53 @@ _fetch_atb_variants() { variants_list=("$(_create_atb_variant_pairs "$atb_variants")") - echo "${variants_list[*]}" | tr ' ' ',' + printf "%s\n" "${variants_list[@]}" +} + +split_array_into_chunks() { + local array=("$@") + local chunk_size=256 + local total_elements=${#array[@]} + local chunks=() + local items + + for ((i = 0; i < total_elements; i += chunk_size)); do + # Format the list of variants in a JSON object suitable for being consumed by GitHub Actions matrix. + # For more info see https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations. + items="$(echo "${array[@]:i:chunk_size}" | tr ' ' ',')" + chunks+=("{\"include\": [${items}]}") + done + + printf "%s\n" "${chunks[@]}" } main() { - local atb_variants - local origin_variants + local variants=() + local items=() + # fetch ATB variants - atb_variants="$(_fetch_atb_variants)" + variants+=("$(_fetch_atb_variants)") # fetch Origin variants - origin_variants="$(_fetch_origin_tasks)" - # merges the two list together. Use `include` keyword for later usage in matrix. - # for more info see https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations. - local merged_variants="{\"include\": [${atb_variants},${origin_variants}]}" - # write in GitHub output - echo "build-variants=${merged_variants}" >> "$GITHUB_OUTPUT" + variants+=("$(_fetch_origin_tasks "$origin_batch")") + + while read -r variant; do + items+=("$variant") + done <<< "$(printf "%s\n" "${variants[@]}")" + + echo "Found ${#items[@]} variants" + + local chunks=() + while read -r chunk; do + chunks+=("$chunk") + done <<< "$(split_array_into_chunks "${items[@]}")" + + local i=1 + for chunk in "${chunks[@]}"; do + # Save to GitHub output + echo "Storing chunk #${i}" + echo "build-variants-${i}=${chunk}" >> "$GITHUB_OUTPUT" + i=$((i + 1)) + done } -main +main "$@" diff --git a/.github/workflows/create_variant.yml b/.github/workflows/create_variant.yml index 671d7e57de..ff3ea7f6be 100644 --- a/.github/workflows/create_variant.yml +++ b/.github/workflows/create_variant.yml @@ -1,3 +1,5 @@ +name: Create DMG Variant + on: workflow_dispatch: inputs: diff --git a/.github/workflows/create_variants.yml b/.github/workflows/create_variants.yml index a2f366a555..5516df6f65 100644 --- a/.github/workflows/create_variants.yml +++ b/.github/workflows/create_variants.yml @@ -36,7 +36,8 @@ jobs: timeout-minutes: 15 outputs: - build-variants: ${{ steps.get-build-variants.outputs.build-variants }} + build-variants-1: ${{ steps.get-build-variants.outputs.build-variants-1 }} + build-variants-2: ${{ steps.get-build-variants.outputs.build-variants-2 }} steps: - name: Check out repository @@ -69,14 +70,35 @@ jobs: path: ${{ github.workspace }}/duckduckgo.dmg retention-days: 1 - create-variants: + create-variants-1: name: Create Variant needs: [set-up-variants, download-dmg-and-upload-artifact] strategy: fail-fast: false - matrix: ${{ fromJSON(needs.set-up-variants.outputs.build-variants) }} + matrix: ${{ fromJSON(needs.set-up-variants.outputs.build-variants-1) }} + uses: ./.github/workflows/create_variant.yml + with: + atb-variant: ${{ matrix.variant }} + origin-variant: ${{ matrix.origin }} + secrets: + APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }} + APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} + APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }} + AWS_ACCESS_KEY_ID_RELEASE_S3: ${{ secrets.AWS_ACCESS_KEY_ID_RELEASE_S3 }} + AWS_SECRET_ACCESS_KEY_RELEASE_S3: ${{ secrets.AWS_SECRET_ACCESS_KEY_RELEASE_S3 }} + MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} + SSH_PRIVATE_KEY_FASTLANE_MATCH: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }} + + create-variants-2: + + name: Create Variant + needs: [set-up-variants, download-dmg-and-upload-artifact, create-variants-1] + + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.set-up-variants.outputs.build-variants-2) }} uses: ./.github/workflows/create_variant.yml with: atb-variant: ${{ matrix.variant }} @@ -93,12 +115,12 @@ jobs: mattermost: name: Send Mattermost message - needs: create-variants + needs: [create-variants-2] runs-on: macos-15 env: - success: ${{ needs.create-variants.result == 'success' }} - failure: ${{ needs.create-variants.result == 'failure' }} + success: ${{ needs.create-variants-2.result == 'success' }} + failure: ${{ needs.create-variants-2.result == 'failure' }} steps: - name: Send Mattermost message