TDL-24640 - Retry ChunkedEncodingError to fix IncompleteRead errors (#42

)

* Add ChunkedEncodingError to backoff; add unittest

* Version Bump + Changelog

* Remove unused library

* Add broad-exception-raised to pylint disable

---------

Co-authored-by: Andy Lu <[email protected]>
Co-authored-by: dsprayberry <[email protected]>
Co-authored-by: bryantgray <[email protected].

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
           name: 'pylint'
           command: |
             source /usr/local/share/virtualenvs/tap-recharge/bin/activate
-            pylint tap_recharge --disable 'missing-module-docstring,missing-function-docstring,missing-class-docstring,no-else-raise,raise-missing-from,inconsistent-return-statements,line-too-long'
+            pylint tap_recharge --disable 'missing-module-docstring,missing-function-docstring,missing-class-docstring,no-else-raise,raise-missing-from,inconsistent-return-statements,line-too-long,broad-exception-raised'
       - run:
           name: 'Unit Tests'
           command: |

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.0.2
+  * Adds ChunkedEncodingError to backoff handling
+  * Adds unittest
+  * [#42](https://github.com/singer-io/tap-recharge/pull/42)
+
 ## 2.0.1
   * Removes deprecated `shopify_customer_id` field
   * Re-orders customers.json

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-recharge',
-      version='2.0.1',
+      version='2.0.2',
       description='Singer.io tap for extracting data from the ReCharge Payments API 2.0',
       author='[email protected]',
       classifiers=['Programming Language :: Python :: 3 :: Only'],

tap_recharge/client.py

@@ -4,7 +4,7 @@
 
 import singer
 from singer import metrics, utils
-from requests.exceptions import Timeout
+from requests.exceptions import Timeout, ChunkedEncodingError
 
 LOGGER = singer.get_logger()
 REQUEST_TIMEOUT = 600
@@ -189,7 +189,7 @@ def __init__(
     # Backoff the request for 5 times when Timeout or Connection error occurs
     @backoff.on_exception(
         backoff.expo,
-        (Timeout, requests.ConnectionError, Server5xxError),
+        (Timeout, requests.ConnectionError, Server5xxError, ChunkedEncodingError),
         max_tries=5,
         factor=2)
     def __enter__(self):
@@ -221,7 +221,7 @@ def check_access_token(self):
     # Added backoff for 5 times when Timeout error occurs
     @backoff.on_exception(
         backoff.expo,
-        (Timeout, Server5xxError, requests.ConnectionError, RechargeRateLimitError),
+        (Timeout, Server5xxError, requests.ConnectionError, RechargeRateLimitError, ChunkedEncodingError),
         max_tries=5,
         factor=2)
     # Call/rate limit: https://docs.rechargepayments.com/docs/api-rate-limits

tests/unittests/test_backoff.py

@@ -1,7 +1,7 @@
 import unittest
 from unittest import mock
 from parameterized import parameterized
-from requests.exceptions import Timeout, ConnectionError
+from requests.exceptions import Timeout, ConnectionError, ChunkedEncodingError
 from tap_recharge.client import RechargeClient, RechargeRateLimitError, Server5xxError
 
 class MockResponse:
@@ -40,6 +40,15 @@ def test_backoff(self, name, test_exception, data, mocked_request, mocked_sleep,
 
         self.assertEqual(mocked_request.call_count, 5)
 
+    @mock.patch('tap_recharge.client.RechargeClient.check_access_token')
+    @mock.patch('time.sleep')
+    @mock.patch('requests.Session.request', side_effect=ChunkedEncodingError)
+    def test_ChunkedEncodingError(self, mocked_request, mocked_sleep, mocked_check_access_token):
+        with self.assertRaises(ChunkedEncodingError) as e:
+            response_json, _ = self.client_obj.request(self.method, self.path, self.url)
+
+        self.assertEqual(mocked_request.call_count, 5)
+
 class TestBackoffCheckAccessToken(unittest.TestCase):
     """Test cases to verify we backoff 5 times for Timeout, ConnectionError and 5XX errors when call 'check_access_token' from 'request'"""