From 71799d24064c997ebf4ce88e6f33bffc845fdf8c Mon Sep 17 00:00:00 2001 From: Charles Beebe Date: Mon, 1 Aug 2016 16:56:42 -0400 Subject: [PATCH 1/3] Add exponential backoff w/ decorrelated jitter. It's the fastest algorithm from the AWS Architecture Blog post ["Exponential Backoff and Jitter"](https://www.awsarchitectureblog.com/2015/03/backoff.html). Closes #58. --- AUTHORS.rst | 1 + retrying.py | 40 ++++++++++++++++++++++++++++++++++++++++ test_retrying.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 42a456c..ed44279 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -32,3 +32,4 @@ Patches and Suggestions - Jonathan Herriott - Job Evers - Cyrus Durgin +- Charles Beebe diff --git a/retrying.py b/retrying.py index bcb7a9d..6b0888a 100644 --- a/retrying.py +++ b/retrying.py @@ -76,6 +76,8 @@ def __init__(self, stop_func=None, wait_func=None, wait_jitter_max=None, + wait_exp_decorr_jitter_base=None, + wait_exp_decorr_jitter_max=None, before_attempts=None, after_attempts=None): @@ -90,9 +92,27 @@ def __init__(self, self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max self._wait_incrementing_max = MAX_WAIT if wait_incrementing_max is None else wait_incrementing_max self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max + self._wait_exp_decorr_jitter_base = wait_exp_decorr_jitter_base + self._wait_exp_decorr_jitter_max = wait_exp_decorr_jitter_max self._before_attempts = before_attempts self._after_attempts = after_attempts + _decorr_jitter_args = [wait_exp_decorr_jitter_base, + wait_exp_decorr_jitter_max] + _any_decorr_jitter_args = any(map(lambda x: x is not None, _decorr_jitter_args)) + _all_decorr_jitter_args = all(map(lambda x: x is not None, _decorr_jitter_args)) + # Make sure only one jitter strategy was selected. + if wait_jitter_max and _any_decorr_jitter_args: + raise ValueError('Choose either random jitter or exponential backoff with\ + decorrelated jitter.') + # Make sure min & max decorr jitter are sane. + if _all_decorr_jitter_args: + if (wait_exp_decorr_jitter_base > wait_exp_decorr_jitter_max): + raise ValueError('wait_exp_decorr_jitter_base must be less than\ + wait_exp_decorr_jitter_max') + if _any_decorr_jitter_args and not _all_decorr_jitter_args: + raise ValueError('wait_exp_decorr_jitter_base and _max must both be specified.') + # TODO add chaining of stop behaviors # stop behavior stop_funcs = [] @@ -126,6 +146,11 @@ def __init__(self, if wait_exponential_multiplier is not None or wait_exponential_max is not None: wait_funcs.append(self.exponential_sleep) + if _all_decorr_jitter_args: + # Initialize previous delay value, which is only needed by this algorithm. + self._previous_delay_ms = wait_exp_decorr_jitter_base + wait_funcs.append(self.exponential_sleep_with_decorrelated_jitter) + if wait_func is not None: self.wait = wait_func @@ -197,6 +222,21 @@ def exponential_sleep(self, previous_attempt_number, delay_since_first_attempt_m result = 0 return result + def exponential_sleep_with_decorrelated_jitter(self, previous_attempt_number, delay_since_first_attempt_ms): + # Fastest, though not cheapest, exponential backoff algorithm from AWS Architecture + # blog: https://www.awsarchitectureblog.com/2015/03/backoff.html: + # + # sleep(n) = min(sleep_max, random(sleep_base, sleep(n-1) * 3)) + # + # for n >=1 with sleep(0) := sleep_base. Implicitly, sleep_min = sleep_base. + result = min(self._wait_exp_decorr_jitter_max, + random.uniform(self._wait_exp_decorr_jitter_base, + self._previous_delay_ms * 3)) + # Hang onto previous delay value, which is not available in the signature of self.wait. + # Note: this is initialized to the base value in the __init__ method. + self._previous_delay_ms = result + return result + @staticmethod def never_reject(result): return False diff --git a/test_retrying.py b/test_retrying.py index 8ce4ac3..2c38a11 100644 --- a/test_retrying.py +++ b/test_retrying.py @@ -123,6 +123,39 @@ def test_exponential_with_max_wait_and_multiplier(self): self.assertEqual(r.wait(7, 0), 50000) self.assertEqual(r.wait(50, 0), 50000) + def test_exponential_with_decorrelated_jitter(self): + wait_base = 1000 + wait_max = 64000 + r = Retrying(wait_exp_decorr_jitter_base=wait_base, + wait_exp_decorr_jitter_max=wait_max) + # Because wait times are decorrelated from the number of attempts, the parameters passed + # to the wait method don't matter. However, for ease of readability, this test + # increments them as if they do. + previous_wait = wait_base + for i in range(10): + wait = r.wait(i, 0) + # No need to worry about the pathological equality case, which is prevented by + # `test_decorr_base_lt_decorr_max`. + self.assertTrue(wait_base <= wait <= wait_max) + previous_wait = wait + + def test_only_one_jitter(self): + with self.assertRaises(ValueError): + r = Retrying(wait_exp_decorr_jitter_base=0, + wait_exp_decorr_jitter_max=1, + wait_jitter_max=2) + + def test_decorr_base_lt_decorr_max(self): + with self.assertRaises(ValueError): + r = Retrying(wait_exp_decorr_jitter_base=1, + wait_exp_decorr_jitter_max=0) + + def test_all_decorr_args_required(self): + with self.assertRaises(ValueError): + r = Retrying(wait_exp_decorr_jitter_base=0) + with self.assertRaises(ValueError): + r = Retrying(wait_exp_decorr_jitter_max=1) + def test_legacy_explicit_wait_type(self): Retrying(wait="exponential_sleep") From e6b23f23c1c49d6029dfc84af7ffc0a361a385a6 Mon Sep 17 00:00:00 2001 From: Charles Beebe Date: Mon, 1 Aug 2016 17:14:34 -0400 Subject: [PATCH 2/3] Fix tests for python 2.6. Stop using self.assertRaises as a context manager. --- test_retrying.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/test_retrying.py b/test_retrying.py index 2c38a11..5d622e4 100644 --- a/test_retrying.py +++ b/test_retrying.py @@ -140,21 +140,28 @@ def test_exponential_with_decorrelated_jitter(self): previous_wait = wait def test_only_one_jitter(self): - with self.assertRaises(ValueError): - r = Retrying(wait_exp_decorr_jitter_base=0, - wait_exp_decorr_jitter_max=1, - wait_jitter_max=2) + kwargs = { + 'wait_exp_decorr_jitter_base':0, + 'wait_exp_decorr_jitter_max':1, + 'wait_jitter_max':2, + } + self.assertRaises(ValueError, Retrying, **kwargs) def test_decorr_base_lt_decorr_max(self): - with self.assertRaises(ValueError): - r = Retrying(wait_exp_decorr_jitter_base=1, - wait_exp_decorr_jitter_max=0) + kwargs = { + 'wait_exp_decorr_jitter_base':1, + 'wait_exp_decorr_jitter_max':0, + } + self.assertRaises(ValueError, Retrying, **kwargs) def test_all_decorr_args_required(self): - with self.assertRaises(ValueError): - r = Retrying(wait_exp_decorr_jitter_base=0) - with self.assertRaises(ValueError): - r = Retrying(wait_exp_decorr_jitter_max=1) + kwargs = { + 'wait_exp_decorr_jitter_base':0, + } + self.assertRaises(ValueError, Retrying, **kwargs) + kwargs = { + 'wait_exp_decorr_jitter_max':1, + } def test_legacy_explicit_wait_type(self): Retrying(wait="exponential_sleep") From b8056f314eb4a8a760c09f6da96271293e671bd0 Mon Sep 17 00:00:00 2001 From: Charles Beebe Date: Thu, 11 Aug 2016 11:54:29 -0400 Subject: [PATCH 3/3] Change comments to docstring. --- retrying.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/retrying.py b/retrying.py index 6b0888a..ea3f0b1 100644 --- a/retrying.py +++ b/retrying.py @@ -223,12 +223,20 @@ def exponential_sleep(self, previous_attempt_number, delay_since_first_attempt_m return result def exponential_sleep_with_decorrelated_jitter(self, previous_attempt_number, delay_since_first_attempt_ms): - # Fastest, though not cheapest, exponential backoff algorithm from AWS Architecture - # blog: https://www.awsarchitectureblog.com/2015/03/backoff.html: - # - # sleep(n) = min(sleep_max, random(sleep_base, sleep(n-1) * 3)) - # - # for n >=1 with sleep(0) := sleep_base. Implicitly, sleep_min = sleep_base. + """ + Return the length of next delay interval. + + :param previous_attempt_number: The previous attempt number. + :param delay_since_first_attempt_ms: The number of milliseconds since the first delay + interval. + + Implements of the fastest, though not cheapest, exponential backoff algorithm from this + AWS Architecture blog post: https://www.awsarchitectureblog.com/2015/03/backoff.html: + + sleep(n) = min(sleep_max, random(sleep_base, sleep(n-1) * 3)) + + for n >=1 with sleep(0) := sleep_base. + """ result = min(self._wait_exp_decorr_jitter_max, random.uniform(self._wait_exp_decorr_jitter_base, self._previous_delay_ms * 3))