From 3006a9c2f17d0767115d0a4fa9a587e5a6cb38ab Mon Sep 17 00:00:00 2001
From: Bart Louwers <bart@emeel.net>
Date: Fri, 1 Nov 2024 03:01:25 +0100
Subject: [PATCH] Upload Android benchmark results to S3 (#2964)

---
 .github/workflows/android-device-test.yml     | 17 ++++-
 .../aws-device-farm/store-test-artifacts.mjs  | 73 +++++++++++++++----
 2 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/android-device-test.yml b/.github/workflows/android-device-test.yml
index 17c40465683..6172f3ffa1f 100644
--- a/.github/workflows/android-device-test.yml
+++ b/.github/workflows/android-device-test.yml
@@ -162,9 +162,20 @@ jobs:
         if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
         run: |
           npm install
-          temp_dir="$(mktemp -d)"
-          node scripts/aws-device-farm/store-test-artifacts.mjs ${{ steps.aws_device_farm_run.outputs.runArn }} "$temp_dir"
-          zip -r test_artifacts.zip "$temp_dir"
+          echo results_dir="$(mktemp -d)" >> "$GITHUB_ENV"
+          node scripts/aws-device-farm/store-test-artifacts.mjs --runArn ${{ steps.aws_device_farm_run.outputs.runArn }} --outputDir ${{ env.results_dir }}
+          zip -r test_artifacts.zip ${{ env.results_dir }}
+      
+      - name: Store Benchmark Results
+        if: matrix.test.name == 'Android Benchmark' && env.run_device_test == 'true'
+        run: |
+          for zipfile in ${{ env.results_dir }}/*.zip; do
+            unzip "$zipfile" -d "${zipfile%.zip}"
+          done
+
+          find "${{ env.results_dir }}" -name 'benchmark_results.json' | while read -r benchmark_json; do
+            aws s3 cp "$benchmark_json" "s3://maplibre-native/android-benchmark-render/$(uuidgen).json"
+          done
 
       - name: Upload Test Artifacts
         if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
diff --git a/scripts/aws-device-farm/store-test-artifacts.mjs b/scripts/aws-device-farm/store-test-artifacts.mjs
index c02991f156c..b69a16d0bad 100644
--- a/scripts/aws-device-farm/store-test-artifacts.mjs
+++ b/scripts/aws-device-farm/store-test-artifacts.mjs
@@ -3,29 +3,72 @@ import { Readable } from "node:stream";
 import { finished } from "node:stream/promises";
 import * as path from "node:path";
 import * as crypto from "node:crypto";
+import { parseArgs } from "node:util";
 
-import { ListArtifactsCommand, ListJobsCommand, ListSuitesCommand } from "@aws-sdk/client-device-farm";
+import { ArtifactType, ListArtifactsCommand, ListJobsCommand, ListSuitesCommand } from "@aws-sdk/client-device-farm";
 import { getDeviceFarmClient } from "./device-farm-client.mjs";
 
+function getArgs() {
+  const {
+    values
+  } = parseArgs({
+    options: {
+      outputDir: {
+        type: "string",
+      },
+      runArn: {
+        type: "string"
+      },
+      testsSuite: {
+       type: "boolean" 
+      },
+      customerArtifacts: {
+        type: "boolean"
+      }
+    },
+  });
+  const { outputDir, runArn } = values;
+  if (typeof outputDir !== 'string') usage();
+
+  if (typeof runArn !== 'string') usage();
+
+  function suitesFilter() {
+    const names = new Set();
+    if (values.testsSuite) names.add("Tests Suite");
+
+    if (names.size === 0) return () => true;
+
+    return (/** @type {string} **/ name) => names.has(name);
+  }
+
+  /** @type {() => ArtifactType[]} **/
+  function getArtifactsToDownload() {
+    if (values.customerArtifacts) return ["CUSTOMER_ARTIFACT"];
+    return ['TESTSPEC_OUTPUT', 'CUSTOMER_ARTIFACT', 'CUSTOMER_ARTIFACT_LOG', 'DEVICE_LOG'];
+  }
+
+  return {
+    outputDir,
+    runArn,
+    suitesFilter,
+    artifactsToDownload: getArtifactsToDownload()
+  }
+}
+
+const { outputDir, runArn, suitesFilter, artifactsToDownload } = getArgs();
+
 /**
  * @returns {never}
  */
 function usage() {
   console.error("Stores artifacts from AWS Device Farm run");
-  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN OUTPUT_DIR`);
+  console.error("Usage: node store-test-artifacts.mjs --outputDir OUTPUT_DIR --runArn RUN_ARN");
+  console.error("Arguments:")
+  console.error("--customerArtifacts: only download customer artifacts");
+  console.error("--testsSuite: only download stuff from Tests Suite");
   process.exit(1);
 }
 
-if (process.argv.length !== 4) usage();
-
-const arn = process.argv.at(2);
-const outputDirArg = process.argv.at(3);
-
-if (typeof arn !== 'string') usage();
-if (typeof outputDirArg !== 'string') usage();
-
-const outputDir = outputDirArg;
-
 if (!fs.existsSync(outputDir)) {
   console.error("Output dir does not exist");
   process.exit(1);
@@ -46,14 +89,14 @@ async function getTestSpecOutput(arn) {
   
   await Promise.all((jobs.jobs || []).map(async (job) => {
     const suites = await deviceFarmClient.send(new ListSuitesCommand({arn: job.arn}));
-    await Promise.all((suites.suites || []).map(async (suite) => {
+    await Promise.all((suites.suites || []).filter(suitesFilter).map(async (suite) => {
       const artifacts = await deviceFarmClient.send(new ListArtifactsCommand({
         arn: suite.arn,
         type: 'FILE'
       }));
       await Promise.all((artifacts.artifacts || []).map(async (artifact) => {
         if (!artifact.name || !artifact.url || !artifact.type) return;
-        if (['TESTSPEC_OUTPUT', 'CUSTOMER_ARTIFACT', 'CUSTOMER_ARTIFACT_LOG', 'DEVICE_LOG'].includes(artifact.type)) {
+        if (artifactsToDownload.includes(artifact.type)) {
           const filename = `${artifact.name.replaceAll(' ', '_')}-${crypto.randomBytes(10).toString('hex')}.${artifact.extension}`;
           const res = await fetch(artifact.url);
           if (!res.ok || !res.body) return;
@@ -66,4 +109,4 @@ async function getTestSpecOutput(arn) {
   }));
 }
 
-await getTestSpecOutput(arn);
+await getTestSpecOutput(runArn);