From 3bb6b7663f3466266a97fa20c0ed091776793b56 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Fri, 29 Oct 2021 11:32:45 +0200 Subject: [PATCH] Retry api calls on 'service unavailable' The keystoneauth1 adapter used as the basis for cinder, glance, and neutron api calls already support to retry on a 503 status call, if the corresponding parameter is passed. Currently, only connection failures are retried, but if the service is behind a load-balancer, that is rather unlikely and instead a Service Unavailable error would be raised. Change-Id: I82cf1d6eecad1262841c49e10d30c1ec5ba26f80 --- nova/conf/glance.py | 14 ++++++++++++++ nova/image/glance.py | 5 ++++- nova/network/neutronv2/api.py | 3 ++- nova/tests/unit/image/test_glance.py | 4 +++- nova/volume/cinder.py | 1 + 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nova/conf/glance.py b/nova/conf/glance.py index b96f437b0f3..cd50d1ef2bd 100644 --- a/nova/conf/glance.py +++ b/nova/conf/glance.py @@ -51,6 +51,20 @@ Specifies the number of retries when uploading / downloading an image to / from glance. 0 means no retries. +"""), + cfg.IntOpt('http_retries', + default=3, + min=0, + help=""" +Number of times glanceclient should retry on any failed http call. + +0 means connection is attempted only once. Setting it to any positive integer +means that on failure connection is retried that many times e.g. setting it +to 3 means total attempts to connect will be 4. + +Possible values: + +* Any integer value. 0 means connection is attempted only once """), cfg.ListOpt('allowed_direct_url_schemes', default=[], diff --git a/nova/image/glance.py b/nova/image/glance.py index 39970d04989..f1d97fe49d9 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -76,7 +76,10 @@ def _glanceclient_from_endpoint(context, endpoint, version): return glanceclient.Client(version, session=sess, auth=auth, endpoint_override=endpoint, - global_request_id=context.global_id) + global_request_id=context.global_id, + connect_retries=CONF.glance.http_retries, + status_code_retries=CONF.glance.http_retries + ) def generate_glance_url(context): diff --git a/nova/network/neutronv2/api.py b/nova/network/neutronv2/api.py index 4580e7788da..02ee58d5107 100644 --- a/nova/network/neutronv2/api.py +++ b/nova/network/neutronv2/api.py @@ -179,7 +179,8 @@ def get_client(context, admin=False): client_args = dict(session=session, auth=auth_plugin, global_request_id=context.global_id, - connect_retries=CONF.neutron.http_retries) + connect_retries=CONF.neutron.http_retries, + status_code_retries=CONF.neutron.http_retries) if CONF.neutron.url: # TODO(efried): Remove in Rocky diff --git a/nova/tests/unit/image/test_glance.py b/nova/tests/unit/image/test_glance.py index e8dae8e27d3..76d97de6144 100644 --- a/nova/tests/unit/image/test_glance.py +++ b/nova/tests/unit/image/test_glance.py @@ -374,8 +374,10 @@ def test_glanceclient_with_ks_session(self, mock_client, mock_load, self.assertEqual(session, glance._SESSION) # Ensure new client created every time client_call = mock.call(2, auth="fake_auth", + connect_retries=glance.CONF.glance.http_retries, endpoint_override=endpoint, session=session, - global_request_id='reqid') + global_request_id='reqid', + status_code_retries=glance.CONF.glance.http_retries) mock_client.assert_has_calls([client_call, client_call]) self.assertEqual("a", result1) self.assertEqual("b", result2) diff --git a/nova/volume/cinder.py b/nova/volume/cinder.py index ebd78db2dc7..7bd62b21b72 100644 --- a/nova/volume/cinder.py +++ b/nova/volume/cinder.py @@ -274,6 +274,7 @@ def cinderclient(context, microversion=None, skip_version_check=False, auth=auth, endpoint_override=endpoint_override, connect_retries=CONF.cinder.http_retries, + status_code_retries=CONF.cinder.http_retries, global_request_id=context.global_id, **service_parameters)