Skip to content

Commit

Permalink
Make openstack retry more configurable
Browse files Browse the repository at this point in the history
  • Loading branch information
@ricolin
ricolin committed Nov 5, 2024
1 parent c02cdb1 commit 5db2443
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 28 deletions.
52 changes: 27 additions & 25 deletions staffeln/common/openstack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@ class RetryHTTPError(tenacity.retry_if_exception):

def __init__(self):
def is_http_error(exception):
# Make sure we don't retry on 404, as not found could be an
# expected status.
# Make sure we don't retry on codes in skip list (default: 404),
# as not found could be an expected status.
skip_codes = CONF.openstack.skip_retry_codes.replace(
' ', '').split(',')
result = (
isinstance(exception, exceptions.HttpException
) and exception.status_code != 404
) and str(exception.status_code) not in skip_codes
)
if result:
LOG.debug(
Expand Down Expand Up @@ -60,9 +62,9 @@ def set_project(self, project):
# user
@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_user_id(self):
user_name = self.conn.config.auth["username"]
Expand All @@ -82,9 +84,9 @@ def get_user_id(self):

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_role_assignments(self, project_id, user_id=None):
filters = {"project": project_id}
Expand All @@ -94,18 +96,18 @@ def get_role_assignments(self, project_id, user_id=None):

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_user(self, user_id):
return self.conn.get_user(name_or_id=user_id)

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_project_member_emails(self, project_id):
members = self.get_role_assignments(project_id)
Expand All @@ -125,18 +127,18 @@ def get_project_member_emails(self, project_id):

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_projects(self):
return self.conn.list_projects()

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_servers(self, project_id=None, all_projects=True, details=True):
if project_id is not None:
Expand All @@ -152,18 +154,18 @@ def get_servers(self, project_id=None, all_projects=True, details=True):

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_volume(self, uuid, project_id):
return self.conn.get_volume_by_id(uuid)

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_backup(self, uuid, project_id=None):
try:
Expand All @@ -190,9 +192,9 @@ def create_backup(

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def delete_backup(self, uuid, project_id=None, force=False):
# Note(Alex): v3 is not supporting force delete?
Expand All @@ -209,9 +211,9 @@ def delete_backup(self, uuid, project_id=None, force=False):

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_backup_quota(self, project_id):
# quota = conn.get_volume_quotas(project_id)
Expand All @@ -220,9 +222,9 @@ def get_backup_quota(self, project_id):

@tenacity.retry(
retry=RetryHTTPError(),
wait=tenacity.wait_exponential(max=30),
wait=tenacity.wait_exponential(max=CONF.openstack.max_retry_interval),
reraise=True,
stop=tenacity.stop_after_delay(CONF.conductor.retry_timeout),
stop=tenacity.stop_after_delay(CONF.openstack.retry_timeout),
)
def get_backup_gigabytes_quota(self, project_id):
# quota = conn.get_volume_quotas(project_id)
Expand Down
30 changes: 29 additions & 1 deletion staffeln/conf/conductor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@
"to define Conductor's configuration."
),
)
openstack_group = cfg.OptGroup(
"openstack",
title="OpenStack Options",
help=_(
"Options under this group are used "
"to define OpneStack related configuration."
),
)

backup_opts = [
cfg.IntOpt(
Expand Down Expand Up @@ -77,11 +85,29 @@
min=0,
help=_("Number of incremental backups between full backups."),
),
]

openstack_opts = [
cfg.IntOpt(
"retry_timeout",
default=300,
min=1,
help=_("The timeout for retry, the unit is one second."),
help=_("The timeout for retry OpenStackSDK HTTP exceptions, "
"the unit is one second."),
),
cfg.IntOpt(
"max_retry_interval",
default=30,
min=0,
help=_("Max time interval for retry OpenStackSDK HTTP exceptions, "
"the unit is one second."),
),
cfg.StrOpt(
"skip_retry_codes",
default="404,",
help=_("A comma separated string that provides a list of HTTP codes "
"to skip retry on for OpenStackSDK HTTP "
"exception. Default only `404` is skipped.")
),
]

Expand Down Expand Up @@ -150,12 +176,14 @@ def register_opts(conf):
conf.register_group(conductor_group)
conf.register_opts(backup_opts, group=conductor_group)
conf.register_opts(rotation_opts, group=conductor_group)
conf.register_opts(openstack_opts, group=openstack_group)
conf.register_opts(coordination_opts, group=coordination_group)


def list_opts():
return {
"DEFAULT": rotation_opts,
conductor_group: backup_opts,
openstack_group: openstack_opts,
coordination_group: coordination_opts,
}
22 changes: 20 additions & 2 deletions staffeln/tests/common/test_openstacksdk.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import tenacity

from staffeln.common import openstack as s_openstack
from staffeln import conf
from staffeln.tests import base


Expand Down Expand Up @@ -46,7 +47,8 @@ def setUp(self):
self.fake_role_assignment2 = mock.MagicMock(user={"id": "bar"})

def _test_http_error(
self, m_func, retry_func, status_code, call_count=1, **kwargs
self, m_func, retry_func, status_code, call_count=1,
**kwargs
):
m_func.side_effect = openstack_exc.HttpException(
http_status=status_code
Expand All @@ -57,7 +59,9 @@ def _test_http_error(
**kwargs,
)
self.assertEqual(status_code, exc.status_code)
if status_code != 404:
skip_retry_codes = conf.CONF.openstack.skip_retry_codes.replace(
' ', '').split(',')
if str(status_code) not in skip_retry_codes:
if call_count == 1:
self.m_sleep.assert_called_once_with(1.0)
else:
Expand All @@ -84,6 +88,20 @@ def test_get_servers(self):
def test_get_servers_non_http_error(self):
self._test_non_http_error(self.m_c.compute.servers, "get_servers")

def test_get_servers_conf_skip_http_error(self):
conf.CONF.set_override('skip_retry_codes', '403,', 'openstack')
self._test_http_error(
self.m_c.compute.servers, "get_servers", status_code=403
)
self.assertEqual('403,', conf.CONF.openstack.skip_retry_codes)

def test_get_servers_conf_skip_http_error_not_hit(self):
conf.CONF.set_override('skip_retry_codes', '403,', 'openstack')
self._test_http_error(
self.m_c.compute.servers, "get_servers", status_code=404
)
self.assertEqual('403,', conf.CONF.openstack.skip_retry_codes)

def test_get_servers_404_http_error(self):
self._test_http_error(
self.m_c.compute.servers, "get_servers", status_code=404
Expand Down

0 comments on commit 5db2443

Please sign in to comment.