Automate Android benchmark running / data processing

.github/workflows/android-device-test.yml

@@ -70,6 +70,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+
       - id: parent_workflow
         run: |
           conclusion=$(curl ${{ github.event.workflow_run.jobs_url }} | jq -r '.jobs[] | select(.name == "android-build").conclusion')
@@ -130,53 +134,41 @@ jobs:
           aws-region: us-west-2
           role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
           role-session-name: ${{ github.run_id }}
-
-      - name: Upload external data
-        if: env.run_device_test == 'true'
+
+      - name: Run ${{ matrix.test.name }} on AWS Device Farm
         run: |
-          export RESULTS_API=${{ secrets.MLN_RESULTS_API }}
-          export AWS_DEVICE_FARM_PROJECT_ARN=${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
-          upload_arn="$(.github/workflows/android-device-test/upload-external-data.sh)"
-          echo external_data_arn="$upload_arn" >> "$GITHUB_ENV"
-
-      - uses: ./.github/actions/aws-device-farm-run
-        id: aws_device_farm_run
-        if: env.run_device_test == 'true'
-        with:
-          name: ${{ matrix.test.name }}
-          appType: ANDROID_APP
-          appFile: ${{ matrix.test.appFile }}
-          testFile: ${{ matrix.test.testFile }}
-          testPackageType: INSTRUMENTATION_TEST_PACKAGE
-          testType: INSTRUMENTATION
-          testFilter: ${{ matrix.test.testFilter }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_ROLE_TO_ASSUME: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
-          AWS_DEVICE_FARM_PROJECT_ARN: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
-          AWS_DEVICE_FARM_DEVICE_POOL_ARN: ${{ matrix.test.devicePool }}
-          externalData: ${{ env.external_data_arn }}
-          testSpecArn: ${{ matrix.test.testSpecArn }}
+          export name="${{ matrix.test.name }}"
+          export appType=ANDROID_APP
+          export appFile="${{ matrix.test.appFile }}"
+          export testFile="${{ matrix.test.testFile }}"
+          export testPackageType=INSTRUMENTATION_TEST_PACKAGE
+          export testType=INSTRUMENTATION
+          export testFilter="${{ matrix.test.testFilter }}"
+          export AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}"
+          export AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+          export AWS_ROLE_TO_ASSUME="${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}"
+          export AWS_DEVICE_FARM_PROJECT_ARN="${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}"
+          export AWS_DEVICE_FARM_DEVICE_POOL_ARN="${{ matrix.test.devicePool }}"
+          export testSpecArn="${{ matrix.test.testSpecArn }}"
+          export wait_for_completion=true
+
+          echo run_arn="$(./scripts/aws-device-farm/aws-device-farm-run.sh)" > "$GITHUB_ENV"
 
       - name: Store Test Artifacts
         if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
         run: |
           npm install
           results_dir="$(mktemp -d)"
           echo results_dir="$results_dir" >> "$GITHUB_ENV"
-          node scripts/aws-device-farm/store-test-artifacts.mjs --runArn ${{ steps.aws_device_farm_run.outputs.runArn }} --outputDir "$results_dir"
-          zip -r test_artifacts.zip ${{ env.results_dir }}
+          node scripts/aws-device-farm/store-test-artifacts.mjs --runArn ${{ env.run_arn }} --outputDir "$results_dir"
+          zip -r test_artifacts.zip "$results_dir"
 
       - name: Store Benchmark Results
         if: matrix.test.name == 'Android Benchmark' && env.run_device_test == 'true'
         run: |
-          for zipfile in ${{ env.results_dir }}/*.zip; do
-            unzip "$zipfile" -d "${zipfile%.zip}"
-          done
-
-          find "${{ env.results_dir }}" -name 'benchmark_results.json' | while read -r benchmark_json; do
-            aws s3 cp "$benchmark_json" "s3://maplibre-native/android-benchmark-render/$(uuidgen).json"
-          done
+          benchmark_results_dir="$(mktemp -d)"
+          node scripts/aws-device-farm/collect-benchmark-outputs.mjs --inputDir "${{ env.results_dir }}" --outputDir "$benchmark_results_dir"
+          node scripts/aws-device-farm/upload-benchmark-outputs-to-s3.mjs --dir "$benchmark_results_dir"
 
       - name: Upload Test Artifacts
         if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'

.github/workflows/ios-device-test.yml

@@ -49,20 +49,23 @@ jobs:
         with:
           files: "${{ matrix.test.xcTestFile }}, ${{ matrix.test.ipaFile }}"
 
-      - uses: ./.github/actions/aws-device-farm-run
+      - name: Run ${{ matrix.test.name }} on AWS Device Farm
         if: steps.check_files.outputs.files_exists == 'true'
-        with:
-          name: ${{ matrix.test.name }}
-          appType: IOS_APP
-          appFile: ${{ matrix.test.ipaFile }}
-          testFile: ${{ matrix.test.xcTestFile }}
-          testPackageType: XCTEST_TEST_PACKAGE
-          testType: XCTEST
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_ROLE_TO_ASSUME: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
-          AWS_DEVICE_FARM_PROJECT_ARN: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
-          AWS_DEVICE_FARM_DEVICE_POOL_ARN: ${{ vars.AWS_DEVICE_FARM_IPHONE_DEVICE_POOL_ARN }}
+        run: |
+          export name="${{ matrix.test.name }}"
+          export appType=IOS_APP
+          export appFile="${{ matrix.test.ipaFile }}"
+          export testFile="${{ matrix.test.xcTestFile }}"
+          export testPackageType=XCTEST_TEST_PACKAGE
+          export testType=XCTEST
+          export AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}"
+          export AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+          export AWS_ROLE_TO_ASSUME="${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}"
+          export AWS_DEVICE_FARM_PROJECT_ARN="${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}"
+          export AWS_DEVICE_FARM_DEVICE_POOL_ARN="${{ vars.AWS_DEVICE_FARM_IPHONE_DEVICE_POOL_ARN }}"
+          export wait_for_completion=true
+
+          ./scripts/aws-device-farm/aws-device-farm-run.sh
 
       - uses: LouisBrunner/[email protected]
         if: always()

.nvmrc

@@ -1 +1 @@
-20.14.0
+23

package.json

@@ -21,7 +21,7 @@
     "@octokit/rest": "^20.1.0",
     "@types/argparse": "^2.0.16",
     "@types/ejs": "^3.1.5",
-    "@types/node": "^20.16.1",
+    "@types/node": "^22.9.0",
     "argparse": "^2.0.1",
     "csscolorparser": "^1.0.3",
     "d3-queue": "3.0.7",