From 9accff3e2ed9485f8e7adfbf03b2dc7e325df8bc Mon Sep 17 00:00:00 2001 From: Victor Andreasson Date: Wed, 3 Apr 2024 22:57:28 +0200 Subject: [PATCH] Add ugly sleeps to the flakiest Espresso tests After some experiments with "retry" test rules and "wait for view with ID" utility functions, I have decided I prefer "dumb" sleeps in the right places. It clearly identifies the problematic parts of the tests, and I believe this has benefits compared to simply re-trying any failing tests without knowing where the problem is. --- .../android/espresso/AgendaFragmentTest.java | 13 ++++-------- .../com/orgzly/android/espresso/BookTest.java | 11 +++++++++- .../orgzly/android/espresso/BooksTest.java | 2 ++ .../android/espresso/ExternalLinksTest.kt | 20 ++++++++++++------- .../android/espresso/InternalLinksTest.kt | 16 ++++++++++++++- .../com/orgzly/android/espresso/MiscTest.java | 6 +++++- .../orgzly/android/espresso/NoteEventsTest.kt | 11 ++++------ .../android/espresso/NoteFragmentTest.kt | 16 +++++++++++++-- .../android/espresso/QueryFragmentTest.java | 2 ++ .../espresso/SavedSearchesFragmentTest.java | 2 ++ .../android/espresso/ShareActivityTest.kt | 2 ++ .../android/espresso/util/EspressoUtils.java | 3 ++- 12 files changed, 75 insertions(+), 29 deletions(-) diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java index 857d873d7..4f084dc9e 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/AgendaFragmentTest.java @@ -22,6 +22,7 @@ import static org.hamcrest.Matchers.not; import android.content.pm.ActivityInfo; +import android.os.SystemClock; import android.widget.DatePicker; import android.widget.TextView; @@ -30,26 +31,17 @@ import com.orgzly.R; import com.orgzly.android.OrgzlyTest; -import com.orgzly.android.RetryTestRule; import com.orgzly.android.prefs.AppPreferences; import com.orgzly.android.ui.main.MainActivity; import org.joda.time.DateTime; import org.junit.After; import org.junit.Ignore; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TestRule; - -import kotlin.jvm.JvmField; public class AgendaFragmentTest extends OrgzlyTest { private ActivityScenario scenario; - @Rule - @JvmField - public final TestRule mRetryTestRule = new RetryTestRule(); - private ActivityScenario defaultSetUp() { testUtils.setupBook("book-one", "First book used for testing\n" + @@ -183,6 +175,7 @@ public void testMoveTaskWithRepeaterToTomorrow() { tomorrow.getMonthOfYear(), tomorrow.getDayOfMonth())); onView(withText(android.R.string.ok)).perform(click()); + SystemClock.sleep(500); onView(withText(R.string.set)).perform(click()); onNotesInAgenda().check(matches(recyclerViewItemCount(23))); } @@ -263,6 +256,7 @@ public void testOpenCorrectNote() { onItemInAgenda(1).perform(click()); + SystemClock.sleep(500); onView(withId(R.id.scroll_view)).check(matches(isDisplayed())); onView(withId(R.id.title_view)).check(matches(withText("Note A"))); } @@ -295,6 +289,7 @@ public void testInactiveDeadline() { scenario = ActivityScenario.launch(MainActivity.class); searchForTextCloseKeyboard("ad.1"); // Overdue, note (scheduled), today + SystemClock.sleep(500); onNotesInAgenda().check(matches(recyclerViewItemCount(3))); } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java index 26550ef96..e94588c37 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/BookTest.java @@ -4,7 +4,6 @@ import static androidx.test.espresso.Espresso.pressBack; import static androidx.test.espresso.action.ViewActions.click; import static androidx.test.espresso.action.ViewActions.longClick; -import static androidx.test.espresso.action.ViewActions.scrollTo; import static androidx.test.espresso.action.ViewActions.swipeUp; import static androidx.test.espresso.assertion.ViewAssertions.doesNotExist; import static androidx.test.espresso.assertion.ViewAssertions.matches; @@ -29,6 +28,7 @@ import android.content.pm.ActivityInfo; import android.widget.DatePicker; +import android.os.SystemClock; import androidx.test.core.app.ActivityScenario; @@ -37,6 +37,7 @@ import com.orgzly.android.prefs.AppPreferences; import com.orgzly.android.ui.main.MainActivity; +import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -107,6 +108,13 @@ public void setUp() throws Exception { onView(allOf(withText("book-name"), isDisplayed())).perform(click()); } + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + scenario.close(); + } + @Test public void testNoteExists() { onNoteInBook(7, R.id.item_head_title_view).check(matches(withText("Note #7."))); @@ -180,6 +188,7 @@ public void testScrollPositionKeptOnRotation() { scenario.onActivity(activity -> activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE)); + SystemClock.sleep(500); onNoteInBook(40).check(matches(isDisplayed())); // Scroll to note scenario.onActivity(activity -> diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/BooksTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/BooksTest.java index b788e7dcb..04d1b75b6 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/BooksTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/BooksTest.java @@ -30,6 +30,7 @@ import android.app.Activity; import android.app.Instrumentation.ActivityResult; import android.content.Intent; +import android.os.SystemClock; import androidx.documentfile.provider.DocumentFile; import androidx.test.core.app.ActivityScenario; @@ -111,6 +112,7 @@ public void testReturnToNonExistentBookByPressingBack() { onView(withId(R.id.fab)).check(matches(not(isDisplayed()))); pressBack(); + SystemClock.sleep(500); onView(withId(R.id.fragment_books_view_flipper)).check(matches(isDisplayed())); onView(allOf(withText("book-2"), withId(R.id.item_book_title))).perform(click()); onView(allOf(withText(R.string.book_does_not_exist_anymore), isDisplayed())).check(doesNotExist()); diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt index ab82e6ab2..a356f22a8 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/ExternalLinksTest.kt @@ -1,6 +1,7 @@ package com.orgzly.android.espresso import android.os.Environment +import android.os.SystemClock import androidx.test.core.app.ActivityScenario import androidx.test.espresso.action.ViewActions.click import androidx.test.espresso.assertion.ViewAssertions.matches @@ -8,7 +9,10 @@ import androidx.test.espresso.matcher.ViewMatchers.withText import com.orgzly.R import com.orgzly.android.App import com.orgzly.android.OrgzlyTest -import com.orgzly.android.espresso.util.EspressoUtils.* +import com.orgzly.android.espresso.util.EspressoUtils.clickClickableSpan +import com.orgzly.android.espresso.util.EspressoUtils.onBook +import com.orgzly.android.espresso.util.EspressoUtils.onNoteInBook +import com.orgzly.android.espresso.util.EspressoUtils.onSnackbar import com.orgzly.android.ui.main.MainActivity import org.hamcrest.Matchers.startsWith import org.junit.Test @@ -53,14 +57,16 @@ class ExternalLinksTest(private val param: Parameter) : OrgzlyTest() { fun testLink() { testUtils.setupBook("book", "* Note\n${param.link}") - ActivityScenario.launch(MainActivity::class.java) + ActivityScenario.launch(MainActivity::class.java).use { + // Open book + onBook(0).perform(click()) - // Open book - onBook(0).perform(click()) + // Click on link + onNoteInBook(1, R.id.item_head_content_view).perform(clickClickableSpan(param.link)) - // Click on link - onNoteInBook(1, R.id.item_head_content_view).perform(clickClickableSpan(param.link)) + SystemClock.sleep(500) - param.check() + param.check() + } } } \ No newline at end of file diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt index 28f24ba3a..c9de03af6 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/InternalLinksTest.kt @@ -1,5 +1,6 @@ package com.orgzly.android.espresso +import android.os.SystemClock import android.widget.TextView import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView @@ -12,12 +13,15 @@ import com.orgzly.android.espresso.util.EspressoUtils.* import com.orgzly.android.ui.main.MainActivity import org.hamcrest.CoreMatchers.allOf import org.hamcrest.CoreMatchers.instanceOf +import org.junit.After import org.junit.Before import org.junit.Ignore import org.junit.Test class InternalLinksTest : OrgzlyTest() { + private lateinit var scenario: ActivityScenario + @Before @Throws(Exception::class) override fun setUp() { @@ -77,15 +81,22 @@ class InternalLinksTest : OrgzlyTest() { """.trimIndent() ) - ActivityScenario.launch(MainActivity::class.java) + scenario = ActivityScenario.launch(MainActivity::class.java) onBook(0).perform(click()) } + @After + override fun tearDown() { + super.tearDown() + scenario.close() + } + @Test fun testDifferentCaseUuidInternalLink() { onNoteInBook(1, R.id.item_head_content_view) .perform(clickClickableSpan("id:bdce923b-C3CD-41ED-B58E-8BDF8BABA54F")) + SystemClock.sleep(500) onView(withId(R.id.title_view)).check(matches(withText("Note [b-2]"))) } @@ -93,6 +104,7 @@ class InternalLinksTest : OrgzlyTest() { fun testDifferentCaseCustomIdInternalLink() { onNoteInBook(2, R.id.item_head_content_view) .perform(clickClickableSpan("#Different case custom id")) + SystemClock.sleep(500) onView(withId(R.id.title_view)).check(matches(withText("Note [b-1]"))) } @@ -100,6 +112,7 @@ class InternalLinksTest : OrgzlyTest() { fun testCustomIdLink() { onNoteInBook(3, R.id.item_head_content_view) .perform(clickClickableSpan("#Link to note in a different book")) + SystemClock.sleep(500) onView(withId(R.id.title_view)).check(matches(withText("Note [b-3]"))) } @@ -123,6 +136,7 @@ class InternalLinksTest : OrgzlyTest() { fun testNonExistentId() { onNoteInBook(6, R.id.item_head_content_view) .perform(clickClickableSpan("id:note-with-this-id-does-not-exist")) + SystemClock.sleep(500) onSnackbar() .check(matches(withText("Note with “ID” property set to “note-with-this-id-does-not-exist” not found"))) } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java index d90892e43..11cc9d199 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/MiscTest.java @@ -232,7 +232,7 @@ public void testNewBookDialogShouldSurviveScreenRotation() { try (ActivityScenario scenario = ActivityScenario.launch(MainActivity.class)) { scenario.onActivity(activity -> activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE)); - + SystemClock.sleep(500); onView(withId(R.id.fab)).perform(click()); onView(withId(R.id.dialog_new_book_container)).check(matches(isDisplayed())); @@ -357,6 +357,7 @@ public void testScheduledWithRepeaterToDoneFromNoteFragment() { try (ActivityScenario ignored = ActivityScenario.launch(MainActivity.class)) { settingsSetDoneKeywords("DONE OLD"); + SystemClock.sleep(500); onView(allOf(withText("book-name"), isDisplayed())).perform(click()); onNoteInBook(1).perform(click()); @@ -461,6 +462,7 @@ public void testMainActivityFragments() { // Search results onView(withId(R.id.drawer_layout)).perform(open()); + SystemClock.sleep(500); onView(withText("Scheduled")).perform(click()); fragmentTest(activity, true, withId(R.id.fragment_query_search_view_flipper)); @@ -496,6 +498,7 @@ public void testReposActivityFragments() { } private void fragmentTest(Activity activity, boolean hasSearchMenuItem, Matcher matcher) { + SystemClock.sleep(500); onView(matcher).check(matches(isDisplayed())); activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT); @@ -692,6 +695,7 @@ public void testCheckboxWithFoldedDrawerBeforeIt() { onView(withId(R.id.item_preface_text_view)).perform(clickClickableSpan("[ ]")); + SystemClock.sleep(500); onView(allOf(withId(R.id.item_preface_text_view), withText(containsString("- [X] Item")))) .check(matches(isDisplayed())); } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt index 921c32eb3..a8b622aae 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/NoteEventsTest.kt @@ -1,5 +1,6 @@ package com.orgzly.android.espresso +import android.os.SystemClock import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView import androidx.test.espresso.action.ViewActions.click @@ -8,25 +9,18 @@ import androidx.test.espresso.assertion.ViewAssertions.matches import androidx.test.espresso.matcher.ViewMatchers.* import com.orgzly.R import com.orgzly.android.OrgzlyTest -import com.orgzly.android.RetryTestRule import com.orgzly.android.espresso.util.EspressoUtils.* import com.orgzly.android.ui.main.MainActivity import com.orgzly.org.datetime.OrgDateTime import org.hamcrest.Matchers.not import org.hamcrest.Matchers.startsWith import org.junit.After -import org.junit.Rule import org.junit.Test -import org.junit.rules.TestRule class NoteEventsTest : OrgzlyTest() { private lateinit var scenario: ActivityScenario - @Rule - @JvmField - val mRetryTestRule: TestRule = RetryTestRule() - private val now: String get() = OrgDateTime(true).toString() @@ -160,6 +154,8 @@ class NoteEventsTest : OrgzlyTest() { onNotesInAgenda().check(matches(recyclerViewItemCount(10))) + SystemClock.sleep(2000) + // Today: deadline onItemInAgenda(1, R.id.item_head_scheduled_text).check(matches(not(isDisplayed()))) onItemInAgenda(1, R.id.item_head_deadline_text).check(matches(isDisplayed())) @@ -226,6 +222,7 @@ class NoteEventsTest : OrgzlyTest() { scenario = ActivityScenario.launch(MainActivity::class.java) searchForTextCloseKeyboard("ad.2") + SystemClock.sleep(500) onNotesInAgenda().check(matches(recyclerViewItemCount(2))) } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt index 4f37fe200..2fc127da2 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/NoteFragmentTest.kt @@ -1,6 +1,7 @@ package com.orgzly.android.espresso import android.content.pm.ActivityInfo +import android.os.SystemClock import android.widget.DatePicker import android.widget.TimePicker import androidx.test.core.app.ActivityScenario @@ -17,6 +18,7 @@ import com.orgzly.android.OrgzlyTest import com.orgzly.android.espresso.util.EspressoUtils.* import com.orgzly.android.ui.main.MainActivity import org.hamcrest.Matchers.* +import org.junit.After import org.junit.Before import org.junit.Test @@ -66,6 +68,12 @@ class NoteFragmentTest : OrgzlyTest() { onBook(0).perform(click()) } + @After + override fun tearDown() { + super.tearDown() + scenario.close() + } + @Test fun testDeleteNote() { onNoteInBook(1).perform(click()) @@ -330,6 +338,7 @@ class NoteFragmentTest : OrgzlyTest() { .check(matches(allOf(withText(userDateTime("[2014-01-01 Wed 20:07]")), isDisplayed()))) onView(withId(R.id.state_button)).perform(click()) onView(withText(R.string.clear)).perform(click()) + SystemClock.sleep(500) onView(withId(R.id.closed_button)).check(matches(not(isDisplayed()))) } @@ -388,6 +397,7 @@ class NoteFragmentTest : OrgzlyTest() { scenario.onActivity { activity -> activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE } + SystemClock.sleep(500) // Give AVD time to complete rotation /* Set time. */ onView(withText(R.string.set)).perform(click()) @@ -447,7 +457,8 @@ class NoteFragmentTest : OrgzlyTest() { } onView(withId(R.id.scroll_view)).perform(swipeUp()) // For small screens - + SystemClock.sleep(500) + onView(allOf(withId(R.id.name), withText("prop-name-1"))).check(matches(isDisplayed())) onView(allOf(withId(R.id.value), withText("prop-value-1"))).check(matches(isDisplayed())) onView(allOf(withId(R.id.name), withText("prop-name-2"))).check(matches(isDisplayed())) @@ -507,7 +518,7 @@ class NoteFragmentTest : OrgzlyTest() { @Test fun testBreadcrumbsPromptWhenCreatingNewNote() { onNoteInBook(1).perform(longClick()) - onActionItemClick(R.id.new_note, R.string.new_note); + onActionItemClick(R.id.new_note, R.string.new_note) onView(withText(R.string.new_under)).perform(click()) onView(withId(R.id.title_edit)).perform(*replaceTextCloseKeyboard("1.1")) onView(withId(R.id.breadcrumbs_text)).perform(clickClickableSpan("Note #1.")) @@ -517,6 +528,7 @@ class NoteFragmentTest : OrgzlyTest() { .inRoot(isDialog()) .check(matches(isDisplayed())) + SystemClock.sleep(500) // If we click too early, the button doesn't yet work... onView(withText(R.string.cancel)).perform(click()) // Title remains the same diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/QueryFragmentTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/QueryFragmentTest.java index 39e2c219a..bdc2b2432 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/QueryFragmentTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/QueryFragmentTest.java @@ -35,6 +35,7 @@ import android.widget.DatePicker; import android.widget.TextView; import android.widget.TimePicker; +import android.os.SystemClock; import androidx.test.core.app.ActivityScenario; @@ -585,6 +586,7 @@ public void testNotTagShouldReturnSomeNotes() { "* Note D\n"); scenario = ActivityScenario.launch(MainActivity.class); + SystemClock.sleep(500); onView(allOf(withText("notebook"), isDisplayed())).perform(click()); searchForTextCloseKeyboard(".t.b"); onView(withId(R.id.fragment_query_search_view_flipper)).check(matches(isDisplayed())); diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java b/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java index d013aceed..709c4b94d 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/SavedSearchesFragmentTest.java @@ -24,6 +24,7 @@ import android.app.Activity; import android.app.Instrumentation; import android.content.Intent; +import android.os.SystemClock; import androidx.documentfile.provider.DocumentFile; import androidx.test.core.app.ActivityScenario; @@ -74,6 +75,7 @@ public void testNewSameNameSavedSearch() { public void testUpdateSameNameSavedSearch() { onView(withId(R.id.fragment_saved_searches_flipper)).check(matches(isDisplayed())); onSavedSearch(0).perform(click()); + SystemClock.sleep(500); onView(withId(R.id.fragment_saved_search_flipper)).check(matches(isDisplayed())); onView(withId(R.id.fragment_saved_search_query)).perform(typeText(" edited")); onView(withId(R.id.done)).perform(click()); // Saved search done diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt b/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt index 720070e89..57271ad6c 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt +++ b/app/src/androidTest/java/com/orgzly/android/espresso/ShareActivityTest.kt @@ -3,6 +3,7 @@ package com.orgzly.android.espresso import android.content.Intent import android.content.pm.ActivityInfo import android.net.Uri +import android.os.SystemClock import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView import androidx.test.espresso.action.ViewActions.click @@ -133,6 +134,7 @@ class ShareActivityTest : OrgzlyTest() { activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_PORTRAIT } + SystemClock.sleep(1000) onView(withId(R.id.done)).perform(click()); // Note done } diff --git a/app/src/androidTest/java/com/orgzly/android/espresso/util/EspressoUtils.java b/app/src/androidTest/java/com/orgzly/android/espresso/util/EspressoUtils.java index b7542a0b3..b474274c1 100644 --- a/app/src/androidTest/java/com/orgzly/android/espresso/util/EspressoUtils.java +++ b/app/src/androidTest/java/com/orgzly/android/espresso/util/EspressoUtils.java @@ -20,6 +20,7 @@ import static org.hamcrest.Matchers.anything; import android.content.res.Resources; +import android.os.SystemClock; import android.text.Spanned; import android.text.style.ClickableSpan; import android.view.KeyEvent; @@ -179,7 +180,7 @@ public static ViewInteraction onSavedSearch(int position) { public static ViewInteraction onRecyclerViewItem(@IdRes int recyclerView, int position, @IdRes int childView) { onView(withId(recyclerView)).perform(RecyclerViewActions.scrollToPosition(position)); - + SystemClock.sleep(500); return onView(new EspressoRecyclerViewMatcher(recyclerView) .atPositionOnView(position, childView)); }