diff --git a/.changes/unreleased/Under the Hood-20241204-185729.yaml b/.changes/unreleased/Under the Hood-20241204-185729.yaml new file mode 100644 index 000000000..276308126 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241204-185729.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add retry logic for retryable exceptions +time: 2024-12-04T18:57:29.925299-08:00 +custom: + Author: versusfacit colin-rogers-dbt + Issue: "960" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index d93847634..a23563c72 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -439,22 +439,18 @@ def open(cls, connection): credentials = connection.credentials - def exponential_backoff(attempt: int): - return attempt * attempt - - retryable_exceptions = [ + retryable_exceptions = ( redshift_connector.OperationalError, redshift_connector.DatabaseError, redshift_connector.DataError, redshift_connector.InterfaceError, - ] + ) open_connection = cls.retry_connection( connection, connect=get_connection_method(credentials), logger=logger, retry_limit=credentials.retries, - retry_timeout=exponential_backoff, retryable_exceptions=retryable_exceptions, ) open_connection.backend_pid = cls._get_backend_pid(open_connection) # type: ignore @@ -496,8 +492,18 @@ def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False): if without_comments == "": continue + retryable_exceptions = ( + redshift_connector.InterfaceError, + redshift_connector.InternalError, + ) + connection, cursor = super().add_query( - query, auto_begin, bindings=bindings, abridge_sql_log=abridge_sql_log + query, + auto_begin, + bindings=bindings, + abridge_sql_log=abridge_sql_log, + retryable_exceptions=retryable_exceptions, + retry_limit=self.profile.credentials.retries, ) if cursor is None: diff --git a/pyproject.toml b/pyproject.toml index 210dbe478..e68aa2607 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters>=1.11,<2.0", "dbt-postgres>=1.8,<1.10", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index ff8076215..c625e9a7f 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -1,3 +1,5 @@ +import redshift_connector + from multiprocessing import get_context from unittest import TestCase, mock @@ -103,7 +105,15 @@ def test_add_query_success(self): mock_add_query.return_value = None, cursor self.adapter.connections.add_query("select * from test3") mock_add_query.assert_called_once_with( - "select * from test3", True, bindings=None, abridge_sql_log=False + "select * from test3", + True, + bindings=None, + abridge_sql_log=False, + retryable_exceptions=( + redshift_connector.InterfaceError, + redshift_connector.InternalError, + ), + retry_limit=1, ) def test_add_query_with_no_cursor(self):