mullvad · niklasberglund · Nov 20, 2024
diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml
@@ -46,6 +46,13 @@ on:
         default: '0'
         required: true
         type: string
+      e2e_tests_infra_flavor:
+        description: >
+          Infra environment to run e2e tests on(prod/stagemole).
+          If set to 'stagemole' test-related artefacts will be uploaded.
+        default: 'stagemole'
+        required: true
+        type: string
   # Build if main is updated to ensure up-to-date caches are available
   push:
     branches: [main]
@@ -341,7 +348,7 @@ jobs:
 
       - name: Build stagemole app
         uses: burrunan/gradle-cache-action@v1
-        if: github.event.inputs.run_firebase_tests == 'true'
+        if: github.event.inputs.e2e_test_repeat != '0'
         with:
           job-id: jdk17
           arguments: assemblePlayStagemoleDebug
@@ -475,9 +482,16 @@ jobs:
           REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
         run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }}
 
+      - name: Pull test report
+        if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+        shell: bash -ieo pipefail {0}
+        env:
+          REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
+        run: ./android/scripts/pull-test-output.sh --test-type ${{ matrix.test-type }}
+
       - name: Upload instrumentation report (${{ matrix.test-type }})
         uses: actions/upload-artifact@v4
-        if: always() && matrix.test-repeat != 0
+        if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
         with:
           name: ${{ matrix.test-type }}-instrumentation-report
           path: ${{ steps.prepare-report-dir.outputs.report_dir }}
@@ -526,7 +540,7 @@ jobs:
 
       - name: Calculate timeout
         id: calculate-timeout
-        run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT
+        run: echo "timeout=$(( ${{ matrix.test-repeat }} * 15 ))" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Run instrumented test script
@@ -536,15 +550,38 @@ jobs:
         env:
           AUTO_FETCH_TEST_HELPER_APKS: true
           TEST_TYPE: e2e
-          BILLING_FLAVOR: oss
-          INFRA_FLAVOR: prod
-          VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }}
+          BILLING_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && 'oss' || 'play' }}
+          INFRA_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor }}
+          PARTNER_AUTH: >
+            github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+            && secrets.STAGEMOLE_PARTNER_AUTH ||''
+          VALID_TEST_ACCOUNT_NUMBER: >
+            github.event.inputs.e2e_tests_infra_flavor == 'prod' &&
+            secrets.ANDROID_PROD_TEST_ACCOUNT || ''
           INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000'
           ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }}
           ENABLE_ACCESS_TO_LOCAL_API_TESTS: true
           REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
         run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }}
 
+      - name: Pull test report
+        if: >
+          always() && matrix.test-repeat != 0 &&
+          github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+        shell: bash -ieo pipefail {0}
+        env:
+          REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
+        run: ./android/scripts/pull-test-output.sh --test-type e2e
+
+      - name: Upload e2e instrumentation report
+        uses: actions/upload-artifact@v4
+        if: >
+          always() && matrix.test-repeat != 0 &&
+          github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+        with:
+          name: e2e-instrumentation-report
+          path: ${{ steps.prepare-report-dir.outputs.report_dir }}
+
   firebase-tests:
     name: Run firebase tests
     if: github.event.inputs.run_firebase_tests == 'true'

diff --git a/android/scripts/pull-test-output.sh b/android/scripts/pull-test-output.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -eu
+
+REPORT_DIR="${REPORT_DIR:-}"
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --test-type)
+            if [[ -n "${2-}" && "$2" =~ ^(app|mockapi|e2e)$ ]]; then
+                TEST_TYPE="$2"
+            else
+                echo "Error: Bad or missing test type. Must be one of: app, mockapi, e2e"
+                exit 1
+            fi
+            shift 2
+            ;;
+        *)
+            echo "Unknown argument: $1"
+            exit 1
+            ;;
+    esac
+done
+
+if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then
+    echo ""
+    echo "Error: The variable REPORT_DIR must be set and the directory must exist."
+    exit 1
+fi
+
+DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE"
+DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments"
+LOCAL_LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt"
+LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots"
+LOCAL_TEST_ARRACHMENTS_PATH="$REPORT_DIR/test-attachments"
+
+echo "Collecting report and produced test attachments..."
+adb logcat -d > "$LOCAL_LOGCAT_FILE_PATH"
+adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" 2>&1 || echo "No screenshots"
+adb pull "$DEVICE_TEST_ATTACHMENTS_PATH" "$LOCAL_TEST_ARRACHMENTS_PATH" 2>&1 || echo "No test attachments"
diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh
@@ -109,10 +109,6 @@ case "$TEST_TYPE" in
         echo ""
         echo "Error: The 'e2e' test type with billing flavor 'play' require infra flavor 'stagemole'."
         exit 1
-    elif [[ $BILLING_FLAVOR == "oss" && $INFRA_FLAVOR != "prod" ]]; then
-        echo ""
-        echo "Error: The 'e2e' test type with billing flavor 'oss' require infra flavor 'prod'."
-        exit 1
     fi
     OPTIONAL_TEST_ARGUMENTS=""
     if [[ -n ${INVALID_TEST_ACCOUNT_NUMBER-} ]]; then
@@ -153,8 +149,6 @@ fi
 GRADLE_ENVIRONMENT_VARIABLES="TEST_E2E_ENABLEACCESSTOLOCALAPITESTS=$ENABLE_ACCESS_TO_LOCAL_API_TESTS"
 
 INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt"
-LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt"
-LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots"
 DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE"
 DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments"
 
@@ -238,10 +232,6 @@ echo "### Checking logs for success message ###"
 if grep -q -E "$LOG_SUCCESS_REGEX" "$INSTRUMENTATION_LOG_FILE_PATH"; then
     echo "Success, no failures!"
 else
-    echo "One or more tests failed, see logs for more details."
-    echo "Collecting report..."
-    adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots"
-    adb logcat -d > "$LOGCAT_FILE_PATH"
     exit 1
 fi
 

diff --git a/.../test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt b/.../test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt
@@ -17,6 +17,7 @@ import net.mullvad.mullvadvpn.test.e2e.constant.DEVICE_LIST_URL
 import net.mullvad.mullvadvpn.test.e2e.constant.PARTNER_ACCOUNT_URL
 import org.json.JSONArray
 import org.json.JSONObject
+import org.junit.jupiter.api.fail
 
 class SimpleMullvadHttpClient(context: Context) {
 
@@ -201,6 +202,10 @@ class SimpleMullvadHttpClient(context: Context) {
 
         private val onErrorResponse = { error: VolleyError ->
             if (error.networkResponse != null) {
+                if (error.networkResponse.statusCode == 429) {
+                    fail("Request failed with response status code 429: Too many requests")
+                }
+
                 Logger.e(
                     "Response returned error message: ${error.message} " +
                         "status code: ${error.networkResponse.statusCode}"