Skip to content

Commit

Permalink
Add ugly sleeps to the flakiest Espresso tests
Browse files Browse the repository at this point in the history
After some experiments with "retry" test rules and "wait for view with
ID" utility functions, I have decided I prefer "dumb" sleeps in the
right places. It clearly identifies the problematic parts of the tests,
and I believe this has benefits compared to simply re-trying any failing
tests without knowing where the problem is.
  • Loading branch information
amberin committed Apr 11, 2024
1 parent b24f00d commit 9accff3
Show file tree
Hide file tree
Showing 12 changed files with 75 additions and 29 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.hamcrest.Matchers.not;

import android.content.pm.ActivityInfo;
import android.os.SystemClock;
import android.widget.DatePicker;
import android.widget.TextView;

Expand All @@ -30,26 +31,17 @@

import com.orgzly.R;
import com.orgzly.android.OrgzlyTest;
import com.orgzly.android.RetryTestRule;
import com.orgzly.android.prefs.AppPreferences;
import com.orgzly.android.ui.main.MainActivity;

import org.joda.time.DateTime;
import org.junit.After;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;

import kotlin.jvm.JvmField;

public class AgendaFragmentTest extends OrgzlyTest {
private ActivityScenario<MainActivity> scenario;

@Rule
@JvmField
public final TestRule mRetryTestRule = new RetryTestRule();

private ActivityScenario<MainActivity> defaultSetUp() {
testUtils.setupBook("book-one",
"First book used for testing\n" +
Expand Down Expand Up @@ -183,6 +175,7 @@ public void testMoveTaskWithRepeaterToTomorrow() {
tomorrow.getMonthOfYear(),
tomorrow.getDayOfMonth()));
onView(withText(android.R.string.ok)).perform(click());
SystemClock.sleep(500);
onView(withText(R.string.set)).perform(click());
onNotesInAgenda().check(matches(recyclerViewItemCount(23)));
}
Expand Down Expand Up @@ -263,6 +256,7 @@ public void testOpenCorrectNote() {

onItemInAgenda(1).perform(click());

SystemClock.sleep(500);
onView(withId(R.id.scroll_view)).check(matches(isDisplayed()));
onView(withId(R.id.title_view)).check(matches(withText("Note A")));
}
Expand Down Expand Up @@ -295,6 +289,7 @@ public void testInactiveDeadline() {
scenario = ActivityScenario.launch(MainActivity.class);
searchForTextCloseKeyboard("ad.1");
// Overdue, note (scheduled), today
SystemClock.sleep(500);
onNotesInAgenda().check(matches(recyclerViewItemCount(3)));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import static androidx.test.espresso.Espresso.pressBack;
import static androidx.test.espresso.action.ViewActions.click;
import static androidx.test.espresso.action.ViewActions.longClick;
import static androidx.test.espresso.action.ViewActions.scrollTo;
import static androidx.test.espresso.action.ViewActions.swipeUp;
import static androidx.test.espresso.assertion.ViewAssertions.doesNotExist;
import static androidx.test.espresso.assertion.ViewAssertions.matches;
Expand All @@ -29,6 +28,7 @@

import android.content.pm.ActivityInfo;
import android.widget.DatePicker;
import android.os.SystemClock;

import androidx.test.core.app.ActivityScenario;

Expand All @@ -37,6 +37,7 @@
import com.orgzly.android.prefs.AppPreferences;
import com.orgzly.android.ui.main.MainActivity;

import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
Expand Down Expand Up @@ -107,6 +108,13 @@ public void setUp() throws Exception {
onView(allOf(withText("book-name"), isDisplayed())).perform(click());
}

@After
@Override
public void tearDown() throws Exception {
super.tearDown();
scenario.close();
}

@Test
public void testNoteExists() {
onNoteInBook(7, R.id.item_head_title_view).check(matches(withText("Note #7.")));
Expand Down Expand Up @@ -180,6 +188,7 @@ public void testScrollPositionKeptOnRotation() {
scenario.onActivity(activity ->
activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE));

SystemClock.sleep(500);
onNoteInBook(40).check(matches(isDisplayed())); // Scroll to note

scenario.onActivity(activity ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import android.app.Activity;
import android.app.Instrumentation.ActivityResult;
import android.content.Intent;
import android.os.SystemClock;

import androidx.documentfile.provider.DocumentFile;
import androidx.test.core.app.ActivityScenario;
Expand Down Expand Up @@ -111,6 +112,7 @@ public void testReturnToNonExistentBookByPressingBack() {
onView(withId(R.id.fab)).check(matches(not(isDisplayed())));
pressBack();

SystemClock.sleep(500);
onView(withId(R.id.fragment_books_view_flipper)).check(matches(isDisplayed()));
onView(allOf(withText("book-2"), withId(R.id.item_book_title))).perform(click());
onView(allOf(withText(R.string.book_does_not_exist_anymore), isDisplayed())).check(doesNotExist());
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
package com.orgzly.android.espresso

import android.os.Environment
import android.os.SystemClock
import androidx.test.core.app.ActivityScenario
import androidx.test.espresso.action.ViewActions.click
import androidx.test.espresso.assertion.ViewAssertions.matches
import androidx.test.espresso.matcher.ViewMatchers.withText
import com.orgzly.R
import com.orgzly.android.App
import com.orgzly.android.OrgzlyTest
import com.orgzly.android.espresso.util.EspressoUtils.*
import com.orgzly.android.espresso.util.EspressoUtils.clickClickableSpan
import com.orgzly.android.espresso.util.EspressoUtils.onBook
import com.orgzly.android.espresso.util.EspressoUtils.onNoteInBook
import com.orgzly.android.espresso.util.EspressoUtils.onSnackbar
import com.orgzly.android.ui.main.MainActivity
import org.hamcrest.Matchers.startsWith
import org.junit.Test
Expand Down Expand Up @@ -53,14 +57,16 @@ class ExternalLinksTest(private val param: Parameter) : OrgzlyTest() {
fun testLink() {
testUtils.setupBook("book", "* Note\n${param.link}")

ActivityScenario.launch(MainActivity::class.java)
ActivityScenario.launch(MainActivity::class.java).use {
// Open book
onBook(0).perform(click())

// Open book
onBook(0).perform(click())
// Click on link
onNoteInBook(1, R.id.item_head_content_view).perform(clickClickableSpan(param.link))

// Click on link
onNoteInBook(1, R.id.item_head_content_view).perform(clickClickableSpan(param.link))
SystemClock.sleep(500)

param.check()
param.check()
}
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.orgzly.android.espresso

import android.os.SystemClock
import android.widget.TextView
import androidx.test.core.app.ActivityScenario
import androidx.test.espresso.Espresso.onView
Expand All @@ -12,12 +13,15 @@ import com.orgzly.android.espresso.util.EspressoUtils.*
import com.orgzly.android.ui.main.MainActivity
import org.hamcrest.CoreMatchers.allOf
import org.hamcrest.CoreMatchers.instanceOf
import org.junit.After
import org.junit.Before
import org.junit.Ignore
import org.junit.Test


class InternalLinksTest : OrgzlyTest() {
private lateinit var scenario: ActivityScenario<MainActivity>

@Before
@Throws(Exception::class)
override fun setUp() {
Expand Down Expand Up @@ -77,29 +81,38 @@ class InternalLinksTest : OrgzlyTest() {
""".trimIndent()
)

ActivityScenario.launch(MainActivity::class.java)
scenario = ActivityScenario.launch(MainActivity::class.java)

onBook(0).perform(click())
}

@After
override fun tearDown() {
super.tearDown()
scenario.close()
}

@Test
fun testDifferentCaseUuidInternalLink() {
onNoteInBook(1, R.id.item_head_content_view)
.perform(clickClickableSpan("id:bdce923b-C3CD-41ED-B58E-8BDF8BABA54F"))
SystemClock.sleep(500)
onView(withId(R.id.title_view)).check(matches(withText("Note [b-2]")))
}

@Test
fun testDifferentCaseCustomIdInternalLink() {
onNoteInBook(2, R.id.item_head_content_view)
.perform(clickClickableSpan("#Different case custom id"))
SystemClock.sleep(500)
onView(withId(R.id.title_view)).check(matches(withText("Note [b-1]")))
}

@Test
fun testCustomIdLink() {
onNoteInBook(3, R.id.item_head_content_view)
.perform(clickClickableSpan("#Link to note in a different book"))
SystemClock.sleep(500)
onView(withId(R.id.title_view)).check(matches(withText("Note [b-3]")))
}

Expand All @@ -123,6 +136,7 @@ class InternalLinksTest : OrgzlyTest() {
fun testNonExistentId() {
onNoteInBook(6, R.id.item_head_content_view)
.perform(clickClickableSpan("id:note-with-this-id-does-not-exist"))
SystemClock.sleep(500)
onSnackbar()
.check(matches(withText("Note with “ID” property set to “note-with-this-id-does-not-exist” not found")))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ public void testNewBookDialogShouldSurviveScreenRotation() {
try (ActivityScenario<MainActivity> scenario = ActivityScenario.launch(MainActivity.class)) {
scenario.onActivity(activity ->
activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE));

SystemClock.sleep(500);
onView(withId(R.id.fab)).perform(click());
onView(withId(R.id.dialog_new_book_container)).check(matches(isDisplayed()));

Expand Down Expand Up @@ -357,6 +357,7 @@ public void testScheduledWithRepeaterToDoneFromNoteFragment() {
try (ActivityScenario<MainActivity> ignored = ActivityScenario.launch(MainActivity.class)) {
settingsSetDoneKeywords("DONE OLD");

SystemClock.sleep(500);
onView(allOf(withText("book-name"), isDisplayed())).perform(click());

onNoteInBook(1).perform(click());
Expand Down Expand Up @@ -461,6 +462,7 @@ public void testMainActivityFragments() {

// Search results
onView(withId(R.id.drawer_layout)).perform(open());
SystemClock.sleep(500);
onView(withText("Scheduled")).perform(click());
fragmentTest(activity, true, withId(R.id.fragment_query_search_view_flipper));

Expand Down Expand Up @@ -496,6 +498,7 @@ public void testReposActivityFragments() {
}

private void fragmentTest(Activity activity, boolean hasSearchMenuItem, Matcher<View> matcher) {
SystemClock.sleep(500);
onView(matcher).check(matches(isDisplayed()));

activity.setRequestedOrientation(ActivityInfo.SCREEN_ORIENTATION_PORTRAIT);
Expand Down Expand Up @@ -692,6 +695,7 @@ public void testCheckboxWithFoldedDrawerBeforeIt() {

onView(withId(R.id.item_preface_text_view)).perform(clickClickableSpan("[ ]"));

SystemClock.sleep(500);
onView(allOf(withId(R.id.item_preface_text_view), withText(containsString("- [X] Item"))))
.check(matches(isDisplayed()));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.orgzly.android.espresso

import android.os.SystemClock
import androidx.test.core.app.ActivityScenario
import androidx.test.espresso.Espresso.onView
import androidx.test.espresso.action.ViewActions.click
Expand All @@ -8,25 +9,18 @@ import androidx.test.espresso.assertion.ViewAssertions.matches
import androidx.test.espresso.matcher.ViewMatchers.*
import com.orgzly.R
import com.orgzly.android.OrgzlyTest
import com.orgzly.android.RetryTestRule
import com.orgzly.android.espresso.util.EspressoUtils.*
import com.orgzly.android.ui.main.MainActivity
import com.orgzly.org.datetime.OrgDateTime
import org.hamcrest.Matchers.not
import org.hamcrest.Matchers.startsWith
import org.junit.After
import org.junit.Rule
import org.junit.Test
import org.junit.rules.TestRule


class NoteEventsTest : OrgzlyTest() {
private lateinit var scenario: ActivityScenario<MainActivity>

@Rule
@JvmField
val mRetryTestRule: TestRule = RetryTestRule()

private val now: String
get() = OrgDateTime(true).toString()

Expand Down Expand Up @@ -160,6 +154,8 @@ class NoteEventsTest : OrgzlyTest() {

onNotesInAgenda().check(matches(recyclerViewItemCount(10)))

SystemClock.sleep(2000)

// Today: deadline
onItemInAgenda(1, R.id.item_head_scheduled_text).check(matches(not(isDisplayed())))
onItemInAgenda(1, R.id.item_head_deadline_text).check(matches(isDisplayed()))
Expand Down Expand Up @@ -226,6 +222,7 @@ class NoteEventsTest : OrgzlyTest() {
scenario = ActivityScenario.launch(MainActivity::class.java)

searchForTextCloseKeyboard("ad.2")
SystemClock.sleep(500)
onNotesInAgenda().check(matches(recyclerViewItemCount(2)))
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.orgzly.android.espresso

import android.content.pm.ActivityInfo
import android.os.SystemClock
import android.widget.DatePicker
import android.widget.TimePicker
import androidx.test.core.app.ActivityScenario
Expand All @@ -17,6 +18,7 @@ import com.orgzly.android.OrgzlyTest
import com.orgzly.android.espresso.util.EspressoUtils.*
import com.orgzly.android.ui.main.MainActivity
import org.hamcrest.Matchers.*
import org.junit.After
import org.junit.Before
import org.junit.Test

Expand Down Expand Up @@ -66,6 +68,12 @@ class NoteFragmentTest : OrgzlyTest() {
onBook(0).perform(click())
}

@After
override fun tearDown() {
super.tearDown()
scenario.close()
}

@Test
fun testDeleteNote() {
onNoteInBook(1).perform(click())
Expand Down Expand Up @@ -330,6 +338,7 @@ class NoteFragmentTest : OrgzlyTest() {
.check(matches(allOf(withText(userDateTime("[2014-01-01 Wed 20:07]")), isDisplayed())))
onView(withId(R.id.state_button)).perform(click())
onView(withText(R.string.clear)).perform(click())
SystemClock.sleep(500)
onView(withId(R.id.closed_button)).check(matches(not(isDisplayed())))
}

Expand Down Expand Up @@ -388,6 +397,7 @@ class NoteFragmentTest : OrgzlyTest() {
scenario.onActivity { activity ->
activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE
}
SystemClock.sleep(500) // Give AVD time to complete rotation

/* Set time. */
onView(withText(R.string.set)).perform(click())
Expand Down Expand Up @@ -447,7 +457,8 @@ class NoteFragmentTest : OrgzlyTest() {
}

onView(withId(R.id.scroll_view)).perform(swipeUp()) // For small screens

SystemClock.sleep(500)

onView(allOf(withId(R.id.name), withText("prop-name-1"))).check(matches(isDisplayed()))
onView(allOf(withId(R.id.value), withText("prop-value-1"))).check(matches(isDisplayed()))
onView(allOf(withId(R.id.name), withText("prop-name-2"))).check(matches(isDisplayed()))
Expand Down Expand Up @@ -507,7 +518,7 @@ class NoteFragmentTest : OrgzlyTest() {
@Test
fun testBreadcrumbsPromptWhenCreatingNewNote() {
onNoteInBook(1).perform(longClick())
onActionItemClick(R.id.new_note, R.string.new_note);
onActionItemClick(R.id.new_note, R.string.new_note)
onView(withText(R.string.new_under)).perform(click())
onView(withId(R.id.title_edit)).perform(*replaceTextCloseKeyboard("1.1"))
onView(withId(R.id.breadcrumbs_text)).perform(clickClickableSpan("Note #1."))
Expand All @@ -517,6 +528,7 @@ class NoteFragmentTest : OrgzlyTest() {
.inRoot(isDialog())
.check(matches(isDisplayed()))

SystemClock.sleep(500) // If we click too early, the button doesn't yet work...
onView(withText(R.string.cancel)).perform(click())

// Title remains the same
Expand Down
Loading

0 comments on commit 9accff3

Please sign in to comment.