From 030cfa5ede91cc868e341540c9da60ffc5c11fcd Mon Sep 17 00:00:00 2001
From: Jonathan Green <jonathan@razzard.com>
Date: Mon, 9 Dec 2024 11:43:20 -0400
Subject: [PATCH] Retry celery sync task on StaleDataError

---
 .../manager/celery/tasks/patron_activity.py      |  6 ++++--
 .../manager/celery/tasks/test_patron_activity.py | 16 ++++++++++++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/palace/manager/celery/tasks/patron_activity.py b/src/palace/manager/celery/tasks/patron_activity.py
index 824ed65ff..0485a870a 100644
--- a/src/palace/manager/celery/tasks/patron_activity.py
+++ b/src/palace/manager/celery/tasks/patron_activity.py
@@ -1,4 +1,5 @@
 from celery import shared_task
+from sqlalchemy.orm.exc import StaleDataError
 
 from palace.manager.api.circulation import PatronActivityCirculationAPI
 from palace.manager.api.circulation_exceptions import PatronAuthorizationFailedException
@@ -80,8 +81,9 @@ def sync_patron_activity(
                     "Marking patron activity as failed."
                 )
                 return
-            except RemoteIntegrationException as e:
-                # This may have been a transient network error with the remote integration. Attempt to retry.
+            except (RemoteIntegrationException, StaleDataError) as e:
+                # This may have been a transient network error with the remote integration or some data
+                # changed while we were processing the sync. Retry the task.
                 retries = task.request.retries
                 if retries < task.max_retries:
                     wait_time = exponential_backoff(retries)
diff --git a/tests/manager/celery/tasks/test_patron_activity.py b/tests/manager/celery/tasks/test_patron_activity.py
index c9a7fa1b6..3d41e4c88 100644
--- a/tests/manager/celery/tasks/test_patron_activity.py
+++ b/tests/manager/celery/tasks/test_patron_activity.py
@@ -1,6 +1,7 @@
 from unittest.mock import MagicMock, create_autospec, patch
 
 import pytest
+from sqlalchemy.orm.exc import StaleDataError
 
 from palace.manager.api.circulation import HoldInfo, LoanInfo
 from palace.manager.api.circulation_exceptions import PatronAuthorizationFailedException
@@ -150,9 +151,20 @@ def test_patron_authorization_failed_exception(
         )
 
     @patch("palace.manager.celery.tasks.patron_activity.exponential_backoff")
-    def test_remote_integration_exception(
+    @pytest.mark.parametrize(
+        "exception",
+        [
+            pytest.param(StaleDataError(), id="StaleDataError"),
+            pytest.param(
+                RemoteIntegrationException("http://test.com", "boom!"),
+                id="RemoteIntegrationException",
+            ),
+        ],
+    )
+    def test_retried_exception(
         self,
         mock_backoff: MagicMock,
+        exception: Exception,
         sync_task_fixture: SyncTaskFixture,
         caplog: pytest.LogCaptureFixture,
     ):
@@ -161,7 +173,7 @@ def test_remote_integration_exception(
 
         mock_activity = create_autospec(
             sync_task_fixture.mock_collection_api.patron_activity,
-            side_effect=RemoteIntegrationException("http://test.com", "boom!"),
+            side_effect=exception,
         )
         sync_task_fixture.mock_collection_api.patron_activity = mock_activity