Retry api calls on 'service unavailable'

nova/conf/glance.py

@@ -51,6 +51,20 @@
 
 Specifies the number of retries when uploading / downloading
 an image to / from glance. 0 means no retries.
+"""),
+    cfg.IntOpt('http_retries',
+               default=3,
+               min=0,
+               help="""
+Number of times glanceclient should retry on any failed http call.
+
+0 means connection is attempted only once. Setting it to any positive integer
+means that on failure connection is retried that many times e.g. setting it
+to 3 means total attempts to connect will be 4.
+
+Possible values:
+
+* Any integer value. 0 means connection is attempted only once
 """),
     cfg.ListOpt('allowed_direct_url_schemes',
         default=[],

nova/image/glance.py

@@ -76,7 +76,10 @@ def _glanceclient_from_endpoint(context, endpoint, version):
 
     return glanceclient.Client(version, session=sess, auth=auth,
                                endpoint_override=endpoint,
-                               global_request_id=context.global_id)
+                               global_request_id=context.global_id,
+                               connect_retries=CONF.glance.http_retries,
+                               status_code_retries=CONF.glance.http_retries
+                               )
 
 
 def generate_glance_url(context):

nova/network/neutronv2/api.py

@@ -179,7 +179,8 @@ def get_client(context, admin=False):
     client_args = dict(session=session,
                        auth=auth_plugin,
                        global_request_id=context.global_id,
-                       connect_retries=CONF.neutron.http_retries)
+                       connect_retries=CONF.neutron.http_retries,
+                       status_code_retries=CONF.neutron.http_retries)
 
     if CONF.neutron.url:
         # TODO(efried): Remove in Rocky

nova/tests/unit/image/test_glance.py

@@ -374,8 +374,10 @@ def test_glanceclient_with_ks_session(self, mock_client, mock_load,
         self.assertEqual(session, glance._SESSION)
         # Ensure new client created every time
         client_call = mock.call(2, auth="fake_auth",
+                connect_retries=glance.CONF.glance.http_retries,
                 endpoint_override=endpoint, session=session,
-                                global_request_id='reqid')
+                global_request_id='reqid',
+                status_code_retries=glance.CONF.glance.http_retries)
         mock_client.assert_has_calls([client_call, client_call])
         self.assertEqual("a", result1)
         self.assertEqual("b", result2)

nova/volume/cinder.py

@@ -274,6 +274,7 @@ def cinderclient(context, microversion=None, skip_version_check=False,
                                 auth=auth,
                                 endpoint_override=endpoint_override,
                                 connect_retries=CONF.cinder.http_retries,
+                                status_code_retries=CONF.cinder.http_retries,
                                 global_request_id=context.global_id,
                                 **service_parameters)