From b067b083b2a1ac419e2cf3bc894cb6e17ecb6435 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Wed, 20 Nov 2024 17:28:17 +0100 Subject: [PATCH] Use stagemole with e2e tests --- .github/workflows/android-app.yml | 49 ++++++++++++++++--- android/scripts/pull-test-output.sh | 40 +++++++++++++++ android/scripts/run-instrumented-tests.sh | 10 ---- .../test/e2e/misc/SimpleMullvadHttpClient.kt | 5 ++ 4 files changed, 88 insertions(+), 16 deletions(-) create mode 100755 android/scripts/pull-test-output.sh diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml index a016fad97e16..8937363932a6 100644 --- a/.github/workflows/android-app.yml +++ b/.github/workflows/android-app.yml @@ -46,6 +46,13 @@ on: default: '0' required: true type: string + e2e_tests_infra_flavor: + description: > + Infra environment to run e2e tests on(prod/stagemole). + If set to 'stagemole' test-related artefacts will be uploaded. + default: 'stagemole' + required: true + type: string # Build if main is updated to ensure up-to-date caches are available push: branches: [main] @@ -341,7 +348,7 @@ jobs: - name: Build stagemole app uses: burrunan/gradle-cache-action@v1 - if: github.event.inputs.run_firebase_tests == 'true' + if: github.event.inputs.e2e_test_repeat != '0' with: job-id: jdk17 arguments: assemblePlayStagemoleDebug @@ -475,9 +482,16 @@ jobs: REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} + - name: Pull test report + if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + shell: bash -ieo pipefail {0} + env: + REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} + run: ./android/scripts/pull-test-output.sh --test-type ${{ matrix.test-type }} + - name: Upload instrumentation report (${{ matrix.test-type }}) uses: actions/upload-artifact@v4 - if: always() && matrix.test-repeat != 0 + if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole' with: name: ${{ matrix.test-type }}-instrumentation-report path: ${{ steps.prepare-report-dir.outputs.report_dir }} @@ -526,7 +540,7 @@ jobs: - name: Calculate timeout id: calculate-timeout - run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT + run: echo "timeout=$(( ${{ matrix.test-repeat }} * 15 ))" >> $GITHUB_OUTPUT shell: bash - name: Run instrumented test script @@ -536,15 +550,38 @@ jobs: env: AUTO_FETCH_TEST_HELPER_APKS: true TEST_TYPE: e2e - BILLING_FLAVOR: oss - INFRA_FLAVOR: prod - VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }} + BILLING_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && 'oss' || 'play' }} + INFRA_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor }} + PARTNER_AUTH: > + github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + && secrets.STAGEMOLE_PARTNER_AUTH ||'' + VALID_TEST_ACCOUNT_NUMBER: > + github.event.inputs.e2e_tests_infra_flavor == 'prod' && + secrets.ANDROID_PROD_TEST_ACCOUNT || '' INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000' ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }} ENABLE_ACCESS_TO_LOCAL_API_TESTS: true REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} + - name: Pull test report + if: > + always() && matrix.test-repeat != 0 && + github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + shell: bash -ieo pipefail {0} + env: + REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} + run: ./android/scripts/pull-test-output.sh --test-type e2e + + - name: Upload e2e instrumentation report + uses: actions/upload-artifact@v4 + if: > + always() && matrix.test-repeat != 0 && + github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + with: + name: e2e-instrumentation-report + path: ${{ steps.prepare-report-dir.outputs.report_dir }} + firebase-tests: name: Run firebase tests if: github.event.inputs.run_firebase_tests == 'true' diff --git a/android/scripts/pull-test-output.sh b/android/scripts/pull-test-output.sh new file mode 100755 index 000000000000..0468e932f3f9 --- /dev/null +++ b/android/scripts/pull-test-output.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -eu + +REPORT_DIR="${REPORT_DIR:-}" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --test-type) + if [[ -n "${2-}" && "$2" =~ ^(app|mockapi|e2e)$ ]]; then + TEST_TYPE="$2" + else + echo "Error: Bad or missing test type. Must be one of: app, mockapi, e2e" + exit 1 + fi + shift 2 + ;; + *) + echo "Unknown argument: $1" + exit 1 + ;; + esac +done + +if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then + echo "" + echo "Error: The variable REPORT_DIR must be set and the directory must exist." + exit 1 +fi + +DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE" +DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments" +LOCAL_LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt" +LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots" +LOCAL_TEST_ARRACHMENTS_PATH="$REPORT_DIR/test-attachments" + +echo "Collecting report and produced test attachments..." +adb logcat -d > "$LOCAL_LOGCAT_FILE_PATH" +adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" 2>&1 || echo "No screenshots" +adb pull "$DEVICE_TEST_ATTACHMENTS_PATH" "$LOCAL_TEST_ARRACHMENTS_PATH" 2>&1 || echo "No test attachments" diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh index 8d835ddb5ab7..1cbe85ea77ec 100755 --- a/android/scripts/run-instrumented-tests.sh +++ b/android/scripts/run-instrumented-tests.sh @@ -109,10 +109,6 @@ case "$TEST_TYPE" in echo "" echo "Error: The 'e2e' test type with billing flavor 'play' require infra flavor 'stagemole'." exit 1 - elif [[ $BILLING_FLAVOR == "oss" && $INFRA_FLAVOR != "prod" ]]; then - echo "" - echo "Error: The 'e2e' test type with billing flavor 'oss' require infra flavor 'prod'." - exit 1 fi OPTIONAL_TEST_ARGUMENTS="" if [[ -n ${INVALID_TEST_ACCOUNT_NUMBER-} ]]; then @@ -153,8 +149,6 @@ fi GRADLE_ENVIRONMENT_VARIABLES="TEST_E2E_ENABLEACCESSTOLOCALAPITESTS=$ENABLE_ACCESS_TO_LOCAL_API_TESTS" INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt" -LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt" -LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots" DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE" DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments" @@ -238,10 +232,6 @@ echo "### Checking logs for success message ###" if grep -q -E "$LOG_SUCCESS_REGEX" "$INSTRUMENTATION_LOG_FILE_PATH"; then echo "Success, no failures!" else - echo "One or more tests failed, see logs for more details." - echo "Collecting report..." - adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots" - adb logcat -d > "$LOGCAT_FILE_PATH" exit 1 fi diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt index b5dcc1d64790..c8ccc60ddb4d 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt @@ -17,6 +17,7 @@ import net.mullvad.mullvadvpn.test.e2e.constant.DEVICE_LIST_URL import net.mullvad.mullvadvpn.test.e2e.constant.PARTNER_ACCOUNT_URL import org.json.JSONArray import org.json.JSONObject +import org.junit.jupiter.api.fail class SimpleMullvadHttpClient(context: Context) { @@ -201,6 +202,10 @@ class SimpleMullvadHttpClient(context: Context) { private val onErrorResponse = { error: VolleyError -> if (error.networkResponse != null) { + if (error.networkResponse.statusCode == 429) { + fail("Request failed with response status code 429: Too many requests") + } + Logger.e( "Response returned error message: ${error.message} " + "status code: ${error.networkResponse.statusCode}"