From 11a9ff4565de4c5ad7ad2ade4530443c486dfc2e Mon Sep 17 00:00:00 2001 From: Victor Andreasson Date: Wed, 3 Apr 2024 22:57:28 +0200 Subject: [PATCH] Add ugly sleeps to the flakiest Espresso tests After some experiments with "retry" test rules, I have decided I prefer "dumb" sleeps in specific places. It clearly identifies the problematic parts of the tests, and I believe this has benefits compared to simply re-trying any failing tests without knowing where the problem is. I also added some more tidying up between tests. --- .../java/com/orgzly/android/OrgzlyTest.java | 4 ++-- .../android/espresso/AgendaFragmentTest.java | 11 ++-------- .../com/orgzly/android/espresso/BookTest.java | 9 ++++++++- .../android/espresso/ExternalLinksTest.kt | 20 ++++++++++++------- .../android/espresso/InternalLinksTest.kt | 16 ++++++++++++++- .../com/orgzly/android/espresso/MiscTest.java | 3 +++ .../orgzly/android/espresso/NoteEventsTest.kt | 10 +++------- .../android/espresso/NoteFragmentTest.kt | 18 ++++++++++++++++- .../espresso/SavedSearchesFragmentTest.java | 2 ++ .../android/espresso/ShareActivityTest.kt | 2 ++ 10 files changed, 67 insertions(+), 28 deletions(-) diff --git a/app/src/androidTest/java/com/orgzly/android/OrgzlyTest.java b/app/src/androidTest/java/com/orgzly/android/OrgzlyTest.java index 972161839..0cb426221 100644 --- a/app/src/androidTest/java/com/orgzly/android/OrgzlyTest.java +++ b/app/src/androidTest/java/com/orgzly/android/OrgzlyTest.java @@ -84,8 +84,6 @@ public void setUp() throws Exception { // localStorage.cleanup(); setupPreferences(); - - dataRepository.clearDatabase(); } @After @@ -93,6 +91,8 @@ public void tearDown() throws Exception { restorePreferences(); database.close(); + + dataRepository.clearDatabase(); } private void setupPreferences() { diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java index 857d873d7..38abce7f3 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java @@ -22,6 +22,7 @@ import static org.hamcrest.Matchers.not; import android.content.pm.ActivityInfo; +import android.os.SystemClock; import android.widget.DatePicker; import android.widget.TextView; @@ -30,26 +31,17 @@ import com.orgzly.R; import com.orgzly.android.OrgzlyTest; -import com.orgzly.android.RetryTestRule; import com.orgzly.android.prefs.AppPreferences; import com.orgzly.android.ui.main.MainActivity; import org.joda.time.DateTime; import org.junit.After; import org.junit.Ignore; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TestRule; - -import kotlin.jvm.JvmField; public class AgendaFragmentTest extends OrgzlyTest { private ActivityScenario scenario; - @Rule - @JvmField - public final TestRule mRetryTestRule = new RetryTestRule(); - private ActivityScenario defaultSetUp() { testUtils.setupBook("book-one", "First book used for testing\n" + @@ -286,6 +278,7 @@ public void testInactiveScheduled() { scenario = ActivityScenario.launch(MainActivity.class); searchForTextCloseKeyboard("ad.1"); // Overdue, note (deadline), today + SystemClock.sleep(500); onNotesInAgenda().check(matches(recyclerViewItemCount(3))); } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java index 26550ef96..d870954f1 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java @@ -4,7 +4,6 @@ import static androidx.test.espresso.Espresso.pressBack; import static androidx.test.espresso.action.ViewActions.click; import static androidx.test.espresso.action.ViewActions.longClick; -import static androidx.test.espresso.action.ViewActions.scrollTo; import static androidx.test.espresso.action.ViewActions.swipeUp; import static androidx.test.espresso.assertion.ViewAssertions.doesNotExist; import static androidx.test.espresso.assertion.ViewAssertions.matches; @@ -37,6 +36,7 @@ import com.orgzly.android.prefs.AppPreferences; import com.orgzly.android.ui.main.MainActivity; +import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -107,6 +107,13 @@ public void setUp() throws Exception { onView(allOf(withText("book-name"), isDisplayed())).perform(click()); } + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + scenario.close(); + } + @Test public void testNoteExists() { onNoteInBook(7, R.id.item_head_title_view).check(matches(withText("Note #7."))); diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt index ab82e6ab2..a356f22a8 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt @@ -1,6 +1,7 @@ package com.orgzly.android.espresso import android.os.Environment +import android.os.SystemClock import androidx.test.core.app.ActivityScenario import androidx.test.espresso.action.ViewActions.click import androidx.test.espresso.assertion.ViewAssertions.matches @@ -8,7 +9,10 @@ import androidx.test.espresso.matcher.ViewMatchers.withText import com.orgzly.R import com.orgzly.android.App import com.orgzly.android.OrgzlyTest -import com.orgzly.android.espresso.util.EspressoUtils.* +import com.orgzly.android.espresso.util.EspressoUtils.clickClickableSpan +import com.orgzly.android.espresso.util.EspressoUtils.onBook +import com.orgzly.android.espresso.util.EspressoUtils.onNoteInBook +import com.orgzly.android.espresso.util.EspressoUtils.onSnackbar import com.orgzly.android.ui.main.MainActivity import org.hamcrest.Matchers.startsWith import org.junit.Test @@ -53,14 +57,16 @@ class ExternalLinksTest(private val param: Parameter) : OrgzlyTest() { fun testLink() { testUtils.setupBook("book", "* Note\n${param.link}") - ActivityScenario.launch(MainActivity::class.java) + ActivityScenario.launch(MainActivity::class.java).use { + // Open book + onBook(0).perform(click()) - // Open book - onBook(0).perform(click()) + // Click on link + onNoteInBook(1, R.id.item_head_content_view).perform(clickClickableSpan(param.link)) - // Click on link - onNoteInBook(1, R.id.item_head_content_view).perform(clickClickableSpan(param.link)) + SystemClock.sleep(500) - param.check() + param.check() + } } } \ No newline at end of file diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt index 28f24ba3a..c9de03af6 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt @@ -1,5 +1,6 @@ package com.orgzly.android.espresso +import android.os.SystemClock import android.widget.TextView import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView @@ -12,12 +13,15 @@ import com.orgzly.android.espresso.util.EspressoUtils.* import com.orgzly.android.ui.main.MainActivity import org.hamcrest.CoreMatchers.allOf import org.hamcrest.CoreMatchers.instanceOf +import org.junit.After import org.junit.Before import org.junit.Ignore import org.junit.Test class InternalLinksTest : OrgzlyTest() { + private lateinit var scenario: ActivityScenario + @Before @Throws(Exception::class) override fun setUp() { @@ -77,15 +81,22 @@ class InternalLinksTest : OrgzlyTest() { """.trimIndent() ) - ActivityScenario.launch(MainActivity::class.java) + scenario = ActivityScenario.launch(MainActivity::class.java) onBook(0).perform(click()) } + @After + override fun tearDown() { + super.tearDown() + scenario.close() + } + @Test fun testDifferentCaseUuidInternalLink() { onNoteInBook(1, R.id.item_head_content_view) .perform(clickClickableSpan("id:bdce923b-C3CD-41ED-B58E-8BDF8BABA54F")) + SystemClock.sleep(500) onView(withId(R.id.title_view)).check(matches(withText("Note [b-2]"))) } @@ -93,6 +104,7 @@ class InternalLinksTest : OrgzlyTest() { fun testDifferentCaseCustomIdInternalLink() { onNoteInBook(2, R.id.item_head_content_view) .perform(clickClickableSpan("#Different case custom id")) + SystemClock.sleep(500) onView(withId(R.id.title_view)).check(matches(withText("Note [b-1]"))) } @@ -100,6 +112,7 @@ class InternalLinksTest : OrgzlyTest() { fun testCustomIdLink() { onNoteInBook(3, R.id.item_head_content_view) .perform(clickClickableSpan("#Link to note in a different book")) + SystemClock.sleep(500) onView(withId(R.id.title_view)).check(matches(withText("Note [b-3]"))) } @@ -123,6 +136,7 @@ class InternalLinksTest : OrgzlyTest() { fun testNonExistentId() { onNoteInBook(6, R.id.item_head_content_view) .perform(clickClickableSpan("id:note-with-this-id-does-not-exist")) + SystemClock.sleep(500) onSnackbar() .check(matches(withText("Note with “ID” property set to “note-with-this-id-does-not-exist” not found"))) } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java index d90892e43..149377414 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java @@ -461,6 +461,7 @@ public void testMainActivityFragments() { // Search results onView(withId(R.id.drawer_layout)).perform(open()); + SystemClock.sleep(500); onView(withText("Scheduled")).perform(click()); fragmentTest(activity, true, withId(R.id.fragment_query_search_view_flipper)); @@ -496,6 +497,7 @@ public void testReposActivityFragments() { } private void fragmentTest(Activity activity, boolean hasSearchMenuItem, Matcher matcher) { + SystemClock.sleep(500); onView(matcher).check(matches(isDisplayed())); activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT); @@ -692,6 +694,7 @@ public void testCheckboxWithFoldedDrawerBeforeIt() { onView(withId(R.id.item_preface_text_view)).perform(clickClickableSpan("[ ]")); + SystemClock.sleep(500); onView(allOf(withId(R.id.item_preface_text_view), withText(containsString("- [X] Item")))) .check(matches(isDisplayed())); } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt index 921c32eb3..e5e0c1251 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt @@ -1,5 +1,6 @@ package com.orgzly.android.espresso +import android.os.SystemClock import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView import androidx.test.espresso.action.ViewActions.click @@ -8,25 +9,18 @@ import androidx.test.espresso.assertion.ViewAssertions.matches import androidx.test.espresso.matcher.ViewMatchers.* import com.orgzly.R import com.orgzly.android.OrgzlyTest -import com.orgzly.android.RetryTestRule import com.orgzly.android.espresso.util.EspressoUtils.* import com.orgzly.android.ui.main.MainActivity import com.orgzly.org.datetime.OrgDateTime import org.hamcrest.Matchers.not import org.hamcrest.Matchers.startsWith import org.junit.After -import org.junit.Rule import org.junit.Test -import org.junit.rules.TestRule class NoteEventsTest : OrgzlyTest() { private lateinit var scenario: ActivityScenario - @Rule - @JvmField - val mRetryTestRule: TestRule = RetryTestRule() - private val now: String get() = OrgDateTime(true).toString() @@ -160,6 +154,8 @@ class NoteEventsTest : OrgzlyTest() { onNotesInAgenda().check(matches(recyclerViewItemCount(10))) + SystemClock.sleep(500) + // Today: deadline onItemInAgenda(1, R.id.item_head_scheduled_text).check(matches(not(isDisplayed()))) onItemInAgenda(1, R.id.item_head_deadline_text).check(matches(isDisplayed())) diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt index 4f37fe200..c1dad7843 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt @@ -1,6 +1,7 @@ package com.orgzly.android.espresso import android.content.pm.ActivityInfo +import android.os.SystemClock import android.widget.DatePicker import android.widget.TimePicker import androidx.test.core.app.ActivityScenario @@ -17,6 +18,7 @@ import com.orgzly.android.OrgzlyTest import com.orgzly.android.espresso.util.EspressoUtils.* import com.orgzly.android.ui.main.MainActivity import org.hamcrest.Matchers.* +import org.junit.After import org.junit.Before import org.junit.Test @@ -66,6 +68,12 @@ class NoteFragmentTest : OrgzlyTest() { onBook(0).perform(click()) } + @After + override fun tearDown() { + super.tearDown() + scenario.close() + } + @Test fun testDeleteNote() { onNoteInBook(1).perform(click()) @@ -330,6 +338,7 @@ class NoteFragmentTest : OrgzlyTest() { .check(matches(allOf(withText(userDateTime("[2014-01-01 Wed 20:07]")), isDisplayed()))) onView(withId(R.id.state_button)).perform(click()) onView(withText(R.string.clear)).perform(click()) + SystemClock.sleep(500) onView(withId(R.id.closed_button)).check(matches(not(isDisplayed()))) } @@ -388,6 +397,7 @@ class NoteFragmentTest : OrgzlyTest() { scenario.onActivity { activity -> activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE } + SystemClock.sleep(500) // Give AVD time to complete rotation /* Set time. */ onView(withText(R.string.set)).perform(click()) @@ -447,7 +457,8 @@ class NoteFragmentTest : OrgzlyTest() { } onView(withId(R.id.scroll_view)).perform(swipeUp()) // For small screens - + SystemClock.sleep(500) + onView(allOf(withId(R.id.name), withText("prop-name-1"))).check(matches(isDisplayed())) onView(allOf(withId(R.id.value), withText("prop-value-1"))).check(matches(isDisplayed())) onView(allOf(withId(R.id.name), withText("prop-name-2"))).check(matches(isDisplayed())) @@ -476,12 +487,15 @@ class NoteFragmentTest : OrgzlyTest() { @Test fun testContentLineCountUpdatedOnNoteUpdate() { + SystemClock.sleep(500) onNoteInBook(1).perform(click()) onView(withId(R.id.content)).perform(scroll()) // For smaller screens onView(withId(R.id.content)).perform(click()) onView(withId(R.id.content_edit)).perform(typeTextIntoFocusedView("a\nb\nc")) onView(withId(R.id.done)).perform(click()) // Note done + SystemClock.sleep(500) onNoteInBook(1, R.id.item_head_fold_button).perform(click()) + SystemClock.sleep(500) onNoteInBook(1, R.id.item_head_title_view).check(matches(withText(endsWith("3")))) } @@ -509,6 +523,7 @@ class NoteFragmentTest : OrgzlyTest() { onNoteInBook(1).perform(longClick()) onActionItemClick(R.id.new_note, R.string.new_note); onView(withText(R.string.new_under)).perform(click()) + SystemClock.sleep(500) onView(withId(R.id.title_edit)).perform(*replaceTextCloseKeyboard("1.1")) onView(withId(R.id.breadcrumbs_text)).perform(clickClickableSpan("Note #1.")) @@ -520,6 +535,7 @@ class NoteFragmentTest : OrgzlyTest() { onView(withText(R.string.cancel)).perform(click()) // Title remains the same + SystemClock.sleep(500) onView(withId(R.id.title_edit)).check(matches(withText("1.1"))) } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java index d013aceed..709c4b94d 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java @@ -24,6 +24,7 @@ import android.app.Activity; import android.app.Instrumentation; import android.content.Intent; +import android.os.SystemClock; import androidx.documentfile.provider.DocumentFile; import androidx.test.core.app.ActivityScenario; @@ -74,6 +75,7 @@ public void testNewSameNameSavedSearch() { public void testUpdateSameNameSavedSearch() { onView(withId(R.id.fragment_saved_searches_flipper)).check(matches(isDisplayed())); onSavedSearch(0).perform(click()); + SystemClock.sleep(500); onView(withId(R.id.fragment_saved_search_flipper)).check(matches(isDisplayed())); onView(withId(R.id.fragment_saved_search_query)).perform(typeText(" edited")); onView(withId(R.id.done)).perform(click()); // Saved search done diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt index 720070e89..57271ad6c 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt @@ -3,6 +3,7 @@ package com.orgzly.android.espresso import android.content.Intent import android.content.pm.ActivityInfo import android.net.Uri +import android.os.SystemClock import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView import androidx.test.espresso.action.ViewActions.click @@ -133,6 +134,7 @@ class ShareActivityTest : OrgzlyTest() { activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_PORTRAIT } + SystemClock.sleep(1000) onView(withId(R.id.done)).perform(click()); // Note done }