From 89ea752eb3fe3378fb7439a0f5287aad37cb0ea0 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Tue, 13 Aug 2024 14:51:14 +0200 Subject: [PATCH] Fix for failing mockapi and e2e tests --- .github/workflows/android-app.yml | 8 ++-- android/BuildInstructions.md | 3 ++ android/gradle/libs.versions.toml | 2 +- android/scripts/run-instrumented-tests.sh | 2 + .../mullvadvpn/service/MullvadVpnService.kt | 3 +- .../common/extension/UiAutomatorExtensions.kt | 12 +++--- .../test/common/interactor/AppInteractor.kt | 9 ----- android/test/e2e/build.gradle.kts | 9 +++++ .../mullvad/mullvadvpn/test/e2e/LoginTest.kt | 4 +- .../mullvad/mullvadvpn/test/e2e/LogoutTest.kt | 13 ------- .../test/e2e/annotations/HighlyRateLimited.kt | 37 +++++++++++++++++++ .../mullvadvpn/test/e2e/constant/Constants.kt | 1 + android/test/firebase/e2e-play-stagemole.yml | 3 ++ android/test/mockapi/build.gradle.kts | 8 ++++ 14 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml index 22a9935ea307..d77a39520923 100644 --- a/.github/workflows/android-app.yml +++ b/.github/workflows/android-app.yml @@ -416,10 +416,9 @@ jobs: - test-type: app path: android/app/build/outputs/apk test-repeat: 1 - # Disabled (test-repeat='0') due to flakiness unless overridden by input. - test-type: mockapi path: android/test/mockapi/build/outputs/apk - test-repeat: ${{ github.event.inputs.mockapi_test_repeat || 0 }} + test-repeat: ${{ github.event.inputs.mockapi_test_repeat || 1 }} steps: - name: Prepare report dir if: ${{ matrix.test-repeat != 0 }} @@ -470,7 +469,8 @@ jobs: instrumented-e2e-tests: name: Run instrumented e2e tests - runs-on: [self-hosted, android-device] + # Temporary workaround for targeting the runner android-runner-v1 + runs-on: [self-hosted, android-device, android-emulator] if: github.event_name == 'schedule' || github.event.inputs.run_e2e_tests == 'true' timeout-minutes: 30 needs: [build-app, build-instrumented-tests] @@ -507,6 +507,7 @@ jobs: INFRA_FLAVOR: prod VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }} INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000' + ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }} REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} run: ./android/scripts/run-instrumented-tests.sh @@ -521,6 +522,7 @@ jobs: clearPackageData=true,\ runnerBuilder=de.mannodermaus.junit5.AndroidJUnit5Builder,\ invalid_test_account_number=0000000000000000,\ + enable_highly_rate_limited_tests=${{ github.event_name == 'schedule' && 'true' || 'false' }},\ partner_auth=${{ secrets.STAGEMOLE_PARTNER_AUTH }}" strategy: fail-fast: false diff --git a/android/BuildInstructions.md b/android/BuildInstructions.md index b6dc1defca1c..fdbea867150c 100644 --- a/android/BuildInstructions.md +++ b/android/BuildInstructions.md @@ -238,3 +238,6 @@ the `ENABLE_IN_APP_VERSION_NOTIFICATIONS` property can be set in `local.properti ``` ENABLE_IN_APP_VERSION_NOTIFICATIONS=false ``` + +### Run tests highly affected by rate limiting +To avoid being rate limited we avoid running tests sending requests that are highly rate limited too often. If you want to run these tests you can set `enable_highly_rate_limited_tests=true` in `local.properties`. The default value is `false`. diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index f603f7005337..90a53d5c6d95 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -71,7 +71,7 @@ android-gradle-aapt = { module = "com.android.tools.build:aapt2" } android-volley = { module = "com.android.volley:volley", version.ref = "android-volley" } # AndroidX -androidx-activity-Compose = { module = "androidx.activity:activity-compose", version.ref = "androidx-activitycompose" } +androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidx-activitycompose" } androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidx-appcompat" } androidx-coresplashscreen = { module = "androidx.core:core-splashscreen", version.ref = "androidx-coresplashscreen" } androidx-espresso = { module = "androidx.test.espresso:espresso-core", version.ref = "androidx-espresso" } diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh index 9ff3ec1b0537..650e7f0feea4 100755 --- a/android/scripts/run-instrumented-tests.sh +++ b/android/scripts/run-instrumented-tests.sh @@ -16,6 +16,7 @@ TEST_SERVICES_URL=https://dl.google.com/android/maven2/androidx/test/services/te PARTNER_AUTH="${PARTNER_AUTH:-}" VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}" INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}" +ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}" REPORT_DIR="${REPORT_DIR:-}" while [[ "$#" -gt 0 ]]; do @@ -130,6 +131,7 @@ case "$TEST_TYPE" in echo "Error: The variable PARTNER_AUTH or VALID_TEST_ACCOUNT_NUMBER must be set." exit 1 fi + OPTIONAL_TEST_ARGUMENTS+=" -e enable_highly_rate_limited_tests $ENABLE_HIGHLY_RATE_LIMITED_TESTS" USE_ORCHESTRATOR="true" PACKAGE_NAME="net.mullvad.mullvadvpn" if [[ "$INFRA_FLAVOR" =~ ^(devmole|stagemole)$ ]]; then diff --git a/android/service/src/main/kotlin/net/mullvad/mullvadvpn/service/MullvadVpnService.kt b/android/service/src/main/kotlin/net/mullvad/mullvadvpn/service/MullvadVpnService.kt index 77c1e92c5e49..6d17b85b6a30 100644 --- a/android/service/src/main/kotlin/net/mullvad/mullvadvpn/service/MullvadVpnService.kt +++ b/android/service/src/main/kotlin/net/mullvad/mullvadvpn/service/MullvadVpnService.kt @@ -16,7 +16,6 @@ import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking -import net.mullvad.mullvadvpn.lib.common.constant.BuildTypes import net.mullvad.mullvadvpn.lib.common.constant.GRPC_SOCKET_FILE_NAMED_ARGUMENT import net.mullvad.mullvadvpn.lib.common.constant.KEY_CONNECT_ACTION import net.mullvad.mullvadvpn.lib.common.constant.KEY_DISCONNECT_ACTION @@ -150,7 +149,7 @@ class MullvadVpnService : TalpidVpnService() { private fun startDaemon() { val apiEndpointConfiguration = - if (Build.TYPE == BuildTypes.DEBUG) { + if (BuildConfig.DEBUG) { intentProvider.getLatestIntent()?.getApiEndpointConfigurationExtras() ?: apiEndpointConfiguration } else { diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt index b3e99eec987b..9f3d727d2906 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt @@ -24,13 +24,11 @@ fun UiDevice.findObjectWithTimeout( wait(Until.hasObject(selector), timeout) - return try { - findObject(selector) - } catch (e: NullPointerException) { - throw IllegalArgumentException( - "No matches for selector within timeout ($timeout): $selector" - ) - } + val foundObject = findObject(selector) + + require(foundObject != null) { "No matches for selector within timeout ($timeout): $selector" } + + return foundObject } fun UiDevice.clickAgreeOnPrivacyDisclaimer() { diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt index 07ea1aeca151..bc2481e55f07 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt @@ -52,15 +52,6 @@ class AppInteractor( ensureLoggedIn() } - fun launchAndCreateAccount() { - launch() - device.clickAgreeOnPrivacyDisclaimer() - device.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove() - waitForLoginPrompt() - attemptCreateAccount() - ensureAccountCreated() - } - fun attemptLogin(accountNumber: String) { val loginObject = device.findObjectWithTimeout(By.clazz("android.widget.EditText")).apply { diff --git a/android/test/e2e/build.gradle.kts b/android/test/e2e/build.gradle.kts index 2e9d19b83cd2..fa59f81940fd 100644 --- a/android/test/e2e/build.gradle.kts +++ b/android/test/e2e/build.gradle.kts @@ -46,6 +46,7 @@ android { put("clearPackageData", "true") addOptionalPropertyAsArgument("valid_test_account_number") addOptionalPropertyAsArgument("invalid_test_account_number") + addOptionalPropertyAsArgument("enable_highly_rate_limited_tests") } } @@ -142,4 +143,12 @@ dependencies { implementation(libs.kotlin.stdlib) androidTestUtil(libs.androidx.test.orchestrator) + + // Needed or else the app crashes when launched + implementation(Dependencies.junit5AndroidTestCompose) + implementation(libs.compose.material3) + + // Need these for forcing later versions of dependencies + implementation(libs.compose.ui) + implementation(libs.androidx.activity.compose) } diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt index 2efb299246ee..bfcc1181dd35 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt @@ -5,8 +5,8 @@ import net.mullvad.mullvadvpn.test.common.constant.LOGIN_FAILURE_TIMEOUT import net.mullvad.mullvadvpn.test.common.extension.clickAgreeOnPrivacyDisclaimer import net.mullvad.mullvadvpn.test.common.extension.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove import net.mullvad.mullvadvpn.test.common.extension.findObjectWithTimeout +import net.mullvad.mullvadvpn.test.e2e.annotations.HighlyRateLimited import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension @@ -27,7 +27,7 @@ class LoginTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { } @Test - @Disabled("Disabled to avoid getting rate-limited.") + @HighlyRateLimited fun testLoginWithInvalidCredentials() { // Given val invalidDummyAccountNumber = accountTestRule.invalidAccountNumber diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LogoutTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LogoutTest.kt index 4d77c2a33a86..7a8c8818f4bc 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LogoutTest.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LogoutTest.kt @@ -23,17 +23,4 @@ class LogoutTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { // Then assertNotNull(device.findObjectWithTimeout(By.text("Login"))) } - - @Test - fun testCreateAccountAndLogout() { - // Given - app.launchAndCreateAccount() - - // When - app.clickAccountCog() - app.clickActionButtonByText("Log out") - - // Then - assertNotNull(device.findObjectWithTimeout(By.text("Login"))) - } } diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt new file mode 100644 index 000000000000..a923e03b461e --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt @@ -0,0 +1,37 @@ +package net.mullvad.mullvadvpn.test.e2e.annotations + +import androidx.test.platform.app.InstrumentationRegistry +import net.mullvad.mullvadvpn.test.e2e.constant.ENABLE_HIGHLY_RATE_LIMITED +import org.junit.jupiter.api.extension.ConditionEvaluationResult +import org.junit.jupiter.api.extension.ExecutionCondition +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.api.extension.ExtensionContext + +/** + * Annotation for tests making use of API endpoints/requests that are highly rate limited such as + * failed login requests. + */ +@Retention(AnnotationRetention.RUNTIME) +@ExtendWith(HighlyRateLimited.ShouldRunWhenSeverelyAffectedByRateLimiting::class) +annotation class HighlyRateLimited { + class ShouldRunWhenSeverelyAffectedByRateLimiting : ExecutionCondition { + override fun evaluateExecutionCondition( + context: ExtensionContext? + ): ConditionEvaluationResult { + val enableHighlyRateLimited = + InstrumentationRegistry.getArguments() + .getString(ENABLE_HIGHLY_RATE_LIMITED) + ?.toBoolean() ?: false + + if (enableHighlyRateLimited) { + return ConditionEvaluationResult.enabled( + "Running test highly affected by rate limiting." + ) + } else { + return ConditionEvaluationResult.disabled( + "Skipping test highly affected by rate limiting." + ) + } + } + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt index baf3dcae3d11..6dbda8f57e7e 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt @@ -4,3 +4,4 @@ const val LOG_TAG = "mullvad-e2e" const val PARTNER_AUTH = "partner_auth" const val VALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "valid_test_account_number" const val INVALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "invalid_test_account_number" +const val ENABLE_HIGHLY_RATE_LIMITED = "enable_highly_rate_limited_tests" diff --git a/android/test/firebase/e2e-play-stagemole.yml b/android/test/firebase/e2e-play-stagemole.yml index a3b48c5699b6..9d3fe92eff9d 100644 --- a/android/test/firebase/e2e-play-stagemole.yml +++ b/android/test/firebase/e2e-play-stagemole.yml @@ -16,3 +16,6 @@ default: - {model: blueline, version: 28, locale: en, orientation: portrait} # pixel 3 - {model: cactus, version: 27, locale: en, orientation: portrait} # redmi 6a - {model: starqlteue, version: 26, locale: en, orientation: portrait} # galaxy s9 + environment-variables: + clearPackageData: "true" + runnerBuilder: "de.mannodermaus.junit5.AndroidJUnit5Builder" diff --git a/android/test/mockapi/build.gradle.kts b/android/test/mockapi/build.gradle.kts index cc13c6dbed46..78994f610aa2 100644 --- a/android/test/mockapi/build.gradle.kts +++ b/android/test/mockapi/build.gradle.kts @@ -88,4 +88,12 @@ dependencies { implementation(libs.mockkWebserver) androidTestUtil(libs.androidx.test.orchestrator) + + // Needed or else the app crashes when launched + implementation(Dependencies.junit5AndroidTestCompose) + implementation(libs.compose.material3) + + // Need these for forcing later versions of dependencies + implementation(libs.compose.ui) + implementation(libs.androidx.activity.compose) }