From 31da5742fdb9baa0006c96c017cef2d0c0b3bc40 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Tue, 20 Aug 2024 17:54:20 +0200 Subject: [PATCH 1/3] Rerun flaky tests in Firebase --- android/test/firebase/e2e-play-stagemole.yml | 1 + android/test/firebase/mockapi-oss.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/android/test/firebase/e2e-play-stagemole.yml b/android/test/firebase/e2e-play-stagemole.yml index 9d3fe92eff9d..24cedac44857 100644 --- a/android/test/firebase/e2e-play-stagemole.yml +++ b/android/test/firebase/e2e-play-stagemole.yml @@ -5,6 +5,7 @@ default: test: android/test/e2e/build/outputs/apk/playStagemole/debug/e2e-play-stagemole-debug.apk timeout: 10m use-orchestrator: true + num-flaky-test-attempts: 1 device: - {model: shiba, version: 34, locale: en, orientation: portrait} # pixel 8 - {model: felix, version: 34, locale: en, orientation: portrait} # pixel fold diff --git a/android/test/firebase/mockapi-oss.yml b/android/test/firebase/mockapi-oss.yml index a1245f1d4f16..d9e180788fe0 100644 --- a/android/test/firebase/mockapi-oss.yml +++ b/android/test/firebase/mockapi-oss.yml @@ -5,6 +5,7 @@ default: test: android/test/mockapi/build/outputs/apk/oss/debug/mockapi-oss-debug.apk timeout: 10m use-orchestrator: true + num-flaky-test-attempts: 1 device: - {model: shiba, version: 34, locale: en, orientation: portrait} - {model: tangorpro, version: 33, locale: en, orientation: portrait} From bbb76b5eef33e7dc3fdec63967fc44383c08b6e7 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Wed, 21 Aug 2024 09:39:01 +0200 Subject: [PATCH 2/3] Handle null response object --- .../kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt | 1 - .../mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt | 9 ++++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt index bfcc1181dd35..505c946cc5f6 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt @@ -5,7 +5,6 @@ import net.mullvad.mullvadvpn.test.common.constant.LOGIN_FAILURE_TIMEOUT import net.mullvad.mullvadvpn.test.common.extension.clickAgreeOnPrivacyDisclaimer import net.mullvad.mullvadvpn.test.common.extension.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove import net.mullvad.mullvadvpn.test.common.extension.findObjectWithTimeout -import net.mullvad.mullvadvpn.test.e2e.annotations.HighlyRateLimited import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt index 734f26653749..eaff4290eddd 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt @@ -194,7 +194,14 @@ class SimpleMullvadHttpClient(context: Context) { "Unable to verify account due to invalid account or connectivity issues." private val onErrorResponse = { error: VolleyError -> - Logger.e("Response returned error status code: ${error.networkResponse.statusCode}") + if (error.networkResponse != null) { + Logger.e( + "Response returned error message: ${error.message} " + + "status code: ${error.networkResponse.statusCode}" + ) + } else { + Logger.e("Response returned error: ${error.message}") + } } } } From 207cc543189b056f1ae3a7dade9b9164b099680f Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Wed, 21 Aug 2024 09:44:54 +0200 Subject: [PATCH 3/3] Disable testLoginWithInvalidCredentials --- .../main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt index 505c946cc5f6..fd829729ad16 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt @@ -6,6 +6,7 @@ import net.mullvad.mullvadvpn.test.common.extension.clickAgreeOnPrivacyDisclaime import net.mullvad.mullvadvpn.test.common.extension.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove import net.mullvad.mullvadvpn.test.common.extension.findObjectWithTimeout import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.RegisterExtension @@ -26,7 +27,7 @@ class LoginTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { } @Test - @HighlyRateLimited + @Disabled("Failed login attempts are highly rate limited and cause test flakiness") fun testLoginWithInvalidCredentials() { // Given val invalidDummyAccountNumber = accountTestRule.invalidAccountNumber