Retry celery sync task on StaleDataError

ThePalaceProject · Dec 9, 2024 · 030cfa5 · 030cfa5
1 parent 0ff6c7d
commit 030cfa5
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 4 deletions.
diff --git a/src/palace/manager/celery/tasks/patron_activity.py b/src/palace/manager/celery/tasks/patron_activity.py
@@ -1,4 +1,5 @@
 from celery import shared_task
+from sqlalchemy.orm.exc import StaleDataError
 
 from palace.manager.api.circulation import PatronActivityCirculationAPI
 from palace.manager.api.circulation_exceptions import PatronAuthorizationFailedException
@@ -80,8 +81,9 @@ def sync_patron_activity(
                     "Marking patron activity as failed."
                 )
                 return
-            except RemoteIntegrationException as e:
-                # This may have been a transient network error with the remote integration. Attempt to retry.
+            except (RemoteIntegrationException, StaleDataError) as e:
+                # This may have been a transient network error with the remote integration or some data
+                # changed while we were processing the sync. Retry the task.
                 retries = task.request.retries
                 if retries < task.max_retries:
                     wait_time = exponential_backoff(retries)

diff --git a/tests/manager/celery/tasks/test_patron_activity.py b/tests/manager/celery/tasks/test_patron_activity.py
@@ -1,6 +1,7 @@
 from unittest.mock import MagicMock, create_autospec, patch
 
 import pytest
+from sqlalchemy.orm.exc import StaleDataError
 
 from palace.manager.api.circulation import HoldInfo, LoanInfo
 from palace.manager.api.circulation_exceptions import PatronAuthorizationFailedException
@@ -150,9 +151,20 @@ def test_patron_authorization_failed_exception(
         )
 
     @patch("palace.manager.celery.tasks.patron_activity.exponential_backoff")
-    def test_remote_integration_exception(
+    @pytest.mark.parametrize(
+        "exception",
+        [
+            pytest.param(StaleDataError(), id="StaleDataError"),
+            pytest.param(
+                RemoteIntegrationException("http://test.com", "boom!"),
+                id="RemoteIntegrationException",
+            ),
+        ],
+    )
+    def test_retried_exception(
         self,
         mock_backoff: MagicMock,
+        exception: Exception,
         sync_task_fixture: SyncTaskFixture,
         caplog: pytest.LogCaptureFixture,
     ):
@@ -161,7 +173,7 @@ def test_remote_integration_exception(
 
         mock_activity = create_autospec(
             sync_task_fixture.mock_collection_api.patron_activity,
-            side_effect=RemoteIntegrationException("http://test.com", "boom!"),
+            side_effect=exception,
         )
         sync_task_fixture.mock_collection_api.patron_activity = mock_activity