make TLS certs reload more robust (#394)

## Issue
In cases where reloading TLS certs via API fails, Opensearch should be
restarted to pick up the new certs. This could happen e.g. after a
network cut, if the API is not available and new TLS certs have been
requested because of a new IP address.

This can be seen [in the current
failures](https://github.com/canonical/opensearch-operator/actions/runs/10361988050/job/28683584556#step:27:5111)
of the integration test `test_ha_networking.py`.

## Solution
If an error occurs when reloading the certs, restart Opensearch to pick
up the new certificates.

lib/charms/opensearch/v0/opensearch_base_charm.py

@@ -727,7 +727,11 @@ def on_tls_conf_set(
 
         # In case of renewal of the unit transport layer cert - restart opensearch
         if renewal and self.is_admin_user_configured() and self.tls.is_fully_configured():
-            self.tls.reload_tls_certificates()
+            try:
+                self.tls.reload_tls_certificates()
+            except OpenSearchHttpError:
+                logger.error("Could not reload TLS certificates via API, will restart.")
+                self._restart_opensearch_event.emit()
 
     def on_tls_relation_broken(self, _: RelationBrokenEvent):
         """As long as all certificates are produced, we don't do anything."""

lib/charms/opensearch/v0/opensearch_tls.py

@@ -655,14 +655,17 @@ def reload_tls_certificates(self):
                 "PUT",
                 url_http,
                 cert_files=(tmp_cert.name, tmp_key.name),
+                retries=3,
             )
             self.charm.opensearch.request(
                 "PUT",
                 url_transport,
                 cert_files=(tmp_cert.name, tmp_key.name),
+                retries=3,
             )
         except OpenSearchHttpError as e:
             logger.error(f"Error reloading TLS certificates via API: {e}")
+            raise
         finally:
             tmp_cert.close()
             tmp_key.close()