feat: sync push (#2413)

Integrate change-tracking, local soil data logic, and soil data push endpoint into full push system. This is comprised of the following notable pieces:

- The `PushDispatcher` component listens for unsynced soil data and, when the app is online, dispatches a push to the server. It will also initialize a retry cycle on failure, which will end the next time the soil sync state changes (i.e., if a push succeed, the user makes new changes, or offline status changes.)
- `soilIdSlice` has been updated to ingest push results using the `applySyncActionResults` method, which handles updating internal recordkeeping based on push operation results. In particular, it only ingests results that are "up-to-date" (the result is for the same revision ID as the record's current revision ID).
- The `sync` model has been updated to include support for sync errors. Records which had an error on the last sync are still considered 'synced' until their next change and retain the last-known 'successful' data.
- `remoteSoilDataActions` contains logic for turning the `ChangeRecords` representing a user's un-synced changes into input for the `Push` mutation.
- `soilDataDiff` contains the bare-minimum logic needed to compare soil data versions, so that we can indicate to the server when a depth interval has been removed. (To be expanded upon in #2283)
- The `sync` file has been split into `revisions`, `records`, and `results` for better separation of concerns and readability.

dev-client/__tests__/integration/models/soilId/soilIdSelectors.test.ts

@@ -0,0 +1,212 @@
+/*
+ * Copyright © 2023-2024 Technology Matters
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see https://www.gnu.org/licenses/.
+ */
+
+import {renderSelectorHook} from '@testing/integration/utils';
+import {cloneDeep} from 'lodash';
+
+import {initialState as accountInitialState} from 'terraso-client-shared/account/accountSlice';
+import {SoilData} from 'terraso-client-shared/soilId/soilIdTypes';
+
+import {
+  selectSyncErrorSites,
+  selectUnsyncedSiteIds,
+  selectUnsyncedSites,
+} from 'terraso-mobile-client/model/soilId/soilIdSelectors';
+import {
+  markEntityError,
+  markEntityModified,
+  markEntitySynced,
+} from 'terraso-mobile-client/model/sync/records';
+import {AppState, useSelector} from 'terraso-mobile-client/store';
+
+const appState = (): AppState => {
+  return {
+    account: {...accountInitialState},
+    map: {userLocation: {accuracyM: null, coords: null}},
+    elevation: {elevationCache: {}},
+    notifications: {messages: {}},
+    preferences: {colorWorkflow: 'MANUAL'},
+    project: {projects: {}},
+    site: {sites: {}},
+    soilId: {
+      matches: {},
+      projectSettings: {},
+      soilSync: {},
+      soilData: {},
+      status: 'ready',
+    },
+  };
+};
+
+const soilData = (): SoilData => {
+  return {
+    depthIntervalPreset: 'CUSTOM',
+    depthDependentData: [],
+    depthIntervals: [],
+  };
+};
+
+describe('selectUnsyncedSites', () => {
+  test('selects unsynced sites only', () => {
+    const state = appState();
+    const now = Date.now();
+    markEntitySynced(state.soilId.soilSync, 'a', {value: soilData()}, now);
+    markEntityModified(state.soilId.soilSync, 'b', now);
+
+    const selected = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      state,
+    );
+
+    expect(selected).toEqual({
+      b: {lastModifiedAt: now, revisionId: 1},
+    });
+  });
+
+  test('returns stable values for input states only', () => {
+    const stateA = appState();
+    markEntityModified(stateA.soilId.soilSync, 'a', Date.now());
+
+    const selectedA1 = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      stateA,
+    );
+    const selectedA2 = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      stateA,
+    );
+
+    const stateB = cloneDeep(stateA);
+    markEntityModified(stateB.soilId.soilSync, 'b', Date.now());
+
+    const selectedB = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      stateB,
+    );
+
+    expect(selectedA1).toBe(selectedA2);
+    expect(selectedA1).not.toBe(selectedB);
+    expect(selectedA2).not.toBe(selectedB);
+  });
+});
+
+describe('selectUnsyncedSiteIds', () => {
+  test('selects unsynced site IDs only, sorted', () => {
+    const state = appState();
+    const now = Date.now();
+    markEntitySynced(state.soilId.soilSync, 'a', {value: soilData()}, now);
+
+    markEntityModified(state.soilId.soilSync, 'c', now);
+    markEntityModified(state.soilId.soilSync, 'b', now);
+
+    const selected = renderSelectorHook(
+      () => useSelector(selectUnsyncedSiteIds),
+      state,
+    );
+
+    expect(selected).toEqual(['b', 'c']);
+  });
+
+  test('returns stable values for input states', () => {
+    const stateA = appState();
+    markEntityModified(stateA.soilId.soilSync, 'a', Date.now());
+
+    const selectedA1 = renderSelectorHook(
+      () => useSelector(selectUnsyncedSiteIds),
+      stateA,
+    );
+    const selectedA2 = renderSelectorHook(
+      () => useSelector(selectUnsyncedSiteIds),
+      stateA,
+    );
+
+    const stateB = cloneDeep(stateA);
+    markEntityModified(stateB.soilId.soilSync, 'b', Date.now());
+
+    const selectedB = renderSelectorHook(
+      () => useSelector(selectUnsyncedSiteIds),
+      stateB,
+    );
+
+    expect(selectedA1).toBe(selectedA2);
+    expect(selectedA1).not.toBe(selectedB);
+    expect(selectedA2).not.toBe(selectedB);
+  });
+});
+
+describe('selectSyncErrorSites', () => {
+  test('selects sync error sites only', () => {
+    const state = appState();
+    const now = Date.now();
+    markEntitySynced(state.soilId.soilSync, 'a', {value: soilData()}, now);
+    markEntityError(
+      state.soilId.soilSync,
+      'b',
+      {revisionId: 1, value: 'DOES_NOT_EXIST'},
+      now,
+    );
+
+    const selected = renderSelectorHook(
+      () => useSelector(selectSyncErrorSites),
+      state,
+    );
+
+    expect(selected).toEqual({
+      b: {
+        lastSyncedAt: now,
+        lastSyncedRevisionId: 1,
+        lastSyncedError: 'DOES_NOT_EXIST',
+      },
+    });
+  });
+
+  test('returns stable values for input states', () => {
+    const stateA = appState();
+    markEntityError(
+      stateA.soilId.soilSync,
+      'a',
+      {value: 'DOES_NOT_EXIST'},
+      Date.now(),
+    );
+
+    const selectedA1 = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      stateA,
+    );
+    const selectedA2 = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      stateA,
+    );
+
+    const stateB = cloneDeep(stateA);
+    markEntityError(
+      stateB.soilId.soilSync,
+      'b',
+      {revisionId: 1, value: 'DOES_NOT_EXIST'},
+      Date.now(),
+    );
+
+    const selectedB = renderSelectorHook(
+      () => useSelector(selectUnsyncedSites),
+      stateB,
+    );
+
+    expect(selectedA1).toBe(selectedA2);
+    expect(selectedA1).not.toBe(selectedB);
+    expect(selectedA2).not.toBe(selectedB);
+  });
+});

dev-client/__tests__/integration/store/sync/PushDispatcher.test.tsx

@@ -0,0 +1,181 @@
+/*
+ * Copyright © 2024 Technology Matters
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see https://www.gnu.org/licenses/.
+ */
+
+import {waitFor} from '@testing-library/react-native';
+import {render} from '@testing/integration/utils';
+
+import * as syncHooks from 'terraso-mobile-client/store/sync/hooks/syncHooks';
+import {
+  PUSH_DEBOUNCE_MS,
+  PUSH_RETRY_INTERVAL_MS,
+  PushDispatcher,
+} from 'terraso-mobile-client/store/sync/PushDispatcher';
+
+jest.mock('terraso-mobile-client/store/sync/hooks/syncHooks', () => {
+  return {
+    useDebouncedIsOffline: jest.fn(),
+    useDebouncedUnsyncedSiteIds: jest.fn(),
+    useIsLoggedIn: jest.fn(),
+    usePushDispatch: jest.fn(),
+    useRetryInterval: jest.fn(),
+  };
+});
+
+describe('PushDispatcher', () => {
+  let useDebouncedIsOffline = jest.mocked(syncHooks.useDebouncedIsOffline);
+  let useIsLoggedIn = jest.mocked(syncHooks.useIsLoggedIn);
+  let useDebouncedUnsyncedSiteIds = jest.mocked(
+    syncHooks.useDebouncedUnsyncedSiteIds,
+  );
+
+  let dispatchPush = jest.fn();
+  let usePushDispatch = jest.mocked(syncHooks.usePushDispatch);
+
+  let beginRetry = jest.fn();
+  let endRetry = jest.fn();
+  let useRetryInterval = jest.mocked(syncHooks.useRetryInterval);
+
+  beforeEach(() => {
+    useDebouncedIsOffline.mockReset();
+    useIsLoggedIn.mockReset();
+    useDebouncedUnsyncedSiteIds.mockReset();
+    useDebouncedUnsyncedSiteIds.mockReset();
+    useDebouncedUnsyncedSiteIds.mockReset();
+
+    dispatchPush.mockReset();
+    usePushDispatch.mockReset();
+    usePushDispatch.mockReturnValue(dispatchPush);
+
+    beginRetry.mockReset();
+    endRetry.mockReset();
+    useRetryInterval.mockReset();
+    jest.mocked(useRetryInterval).mockReturnValue({
+      beginRetry: beginRetry,
+      endRetry: endRetry,
+    });
+  });
+
+  test('uses correct interval for debounces', async () => {
+    render(<PushDispatcher />);
+
+    expect(useDebouncedIsOffline).toHaveBeenCalledWith(PUSH_DEBOUNCE_MS);
+    expect(useDebouncedUnsyncedSiteIds).toHaveBeenCalledWith(PUSH_DEBOUNCE_MS);
+  });
+
+  test('uses correct interval for retry', async () => {
+    render(<PushDispatcher />);
+
+    expect(useRetryInterval).toHaveBeenCalledWith(
+      PUSH_RETRY_INTERVAL_MS,
+      dispatchPush,
+    );
+  });
+
+  test('uses correct site IDs for push dispatch', async () => {
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+
+    render(<PushDispatcher />);
+
+    expect(usePushDispatch).toHaveBeenCalledWith(['abcd']);
+  });
+
+  test('does not dispatch or retry by default', async () => {
+    useIsLoggedIn.mockReturnValue(false);
+    useDebouncedIsOffline.mockReturnValue(true);
+    useDebouncedUnsyncedSiteIds.mockReturnValue([]);
+
+    render(<PushDispatcher />);
+
+    expect(dispatchPush).toHaveBeenCalledTimes(0);
+    expect(beginRetry).toHaveBeenCalledTimes(0);
+    expect(endRetry).toHaveBeenCalledTimes(0);
+  });
+
+  test('dispatches an initial push when conditions are met', async () => {
+    useIsLoggedIn.mockReturnValue(true);
+    useDebouncedIsOffline.mockReturnValue(false);
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+
+    dispatchPush.mockResolvedValue({payload: {}});
+    render(<PushDispatcher />);
+
+    expect(dispatchPush).toHaveBeenCalledTimes(1);
+    expect(beginRetry).toHaveBeenCalledTimes(0);
+    expect(endRetry).toHaveBeenCalledTimes(0);
+  });
+
+  test('begins retry when push has error', async () => {
+    useIsLoggedIn.mockReturnValue(true);
+    useDebouncedIsOffline.mockReturnValue(false);
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+
+    dispatchPush.mockResolvedValue({payload: {error: 'error'}});
+    render(<PushDispatcher />);
+
+    await waitFor(() => expect(beginRetry).toHaveBeenCalledTimes(1));
+  });
+
+  test('begins retry when push is rejected', async () => {
+    useIsLoggedIn.mockReturnValue(true);
+    useDebouncedIsOffline.mockReturnValue(false);
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+
+    dispatchPush.mockRejectedValue('error');
+    render(<PushDispatcher />);
+
+    await waitFor(() => expect(beginRetry).toHaveBeenCalledTimes(1));
+  });
+
+  test('ends retry when logged-in changes', async () => {
+    useIsLoggedIn.mockReturnValue(true);
+    useDebouncedIsOffline.mockReturnValue(false);
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+    dispatchPush.mockRejectedValue('error');
+    const handle = render(<PushDispatcher />);
+
+    useIsLoggedIn.mockReturnValue(false);
+    handle.rerender(<PushDispatcher />);
+
+    await waitFor(() => expect(endRetry).toHaveBeenCalledTimes(1));
+  });
+
+  test('ends retry when online changes', async () => {
+    useIsLoggedIn.mockReturnValue(true);
+    useDebouncedIsOffline.mockReturnValue(false);
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+    dispatchPush.mockRejectedValue('error');
+    const handle = render(<PushDispatcher />);
+
+    useDebouncedIsOffline.mockReturnValue(true);
+    handle.rerender(<PushDispatcher />);
+
+    await waitFor(() => expect(endRetry).toHaveBeenCalledTimes(1));
+  });
+
+  test('ends retry when unsynced ids changes', async () => {
+    useIsLoggedIn.mockReturnValue(true);
+    useDebouncedIsOffline.mockReturnValue(false);
+    useDebouncedUnsyncedSiteIds.mockReturnValue(['abcd']);
+    dispatchPush.mockRejectedValue('error');
+    const handle = render(<PushDispatcher />);
+
+    useDebouncedUnsyncedSiteIds.mockReturnValue([]);
+    handle.rerender(<PushDispatcher />);
+
+    await waitFor(() => expect(endRetry).toHaveBeenCalledTimes(1));
+  });
+});