update error processing

lib/charms/mongodb/v0/helpers.py

@@ -7,7 +7,7 @@
 import secrets
 import string
 import subprocess
-from typing import List, Optional, Union
+from typing import List
 
 from charms.mongodb.v0.mongodb import MongoDBConfiguration, MongoDBConnection
 from ops.model import (
@@ -234,25 +234,6 @@ def copy_licenses_to_unit():
     )
 
 
-_StrOrBytes = Union[str, bytes]
-
-
-def process_pbm_error(error_string: Optional[_StrOrBytes]) -> str:
-    """Parses pbm error string and returns a user friendly message."""
-    message = "couldn't configure s3 backup option"
-    if not error_string:
-        return message
-    if type(error_string) == bytes:
-        error_string = error_string.decode("utf-8")
-    if "status code: 403" in error_string:  # type: ignore
-        message = "s3 credentials are incorrect."
-    elif "status code: 404" in error_string:  # type: ignore
-        message = "s3 configurations are incompatible."
-    elif "status code: 301" in error_string:  # type: ignore
-        message = "s3 configurations are incompatible."
-    return message
-
-
 def current_pbm_op(pbm_status: str) -> str:
     """Parses pbm status for the operation that pbm is running."""
     pbm_status = json.loads(pbm_status)

lib/charms/mongodb/v0/mongodb_backups.py

@@ -13,14 +13,10 @@
 import re
 import subprocess
 import time
-from typing import Dict, List
+from typing import Dict, List, Optional, Union
 
 from charms.data_platform_libs.v0.s3 import CredentialsChangedEvent, S3Requirer
-from charms.mongodb.v0.helpers import (
-    current_pbm_op,
-    process_pbm_error,
-    process_pbm_status,
-)
+from charms.mongodb.v0.helpers import current_pbm_op, process_pbm_status
 from charms.operator_libs_linux.v1 import snap
 from ops.framework import Object
 from ops.model import BlockedStatus, MaintenanceStatus, StatusBase, WaitingStatus
@@ -316,7 +312,7 @@ def _configure_pbm_options(self, event) -> None:
             ),
             return
         except ExecError as e:
-            self.charm.unit.status = BlockedStatus(process_pbm_error(e.stdout))
+            self.charm.unit.status = BlockedStatus(self.process_pbm_error(e.stdout))
             return
         except subprocess.CalledProcessError as e:
             logger.error("Syncing configurations failed: %s", str(e))
@@ -418,7 +414,7 @@ def _wait_pbm_status(self) -> None:
                         )
                         raise ResyncError
                 except ExecError as e:
-                    self.charm.unit.status = BlockedStatus(process_pbm_error(e.stdout))
+                    self.charm.unit.status = BlockedStatus(self.process_pbm_error(e.stdout))
 
     def _get_pbm_status(self) -> StatusBase:
         """Retrieve pbm status."""
@@ -428,15 +424,14 @@ def _get_pbm_status(self) -> StatusBase:
         try:
             previous_pbm_status = self.charm.unit.status
             pbm_status = self.charm.run_pbm_command(PBM_STATUS_CMD)
+
+            # pbm errors are outputted in json and do not raise CLI errors
+            pbm_error = self.process_pbm_error(pbm_status)
+            if pbm_error:
+                return BlockedStatus(pbm_error)
+
             self._log_backup_restore_result(pbm_status, previous_pbm_status)
             return process_pbm_status(pbm_status)
-        except ExecError as e:
-            logger.error(f"Failed to get pbm status. {e}")
-            return BlockedStatus(process_pbm_error(e.stdout))
-        except subprocess.CalledProcessError as e:
-            # pbm pipes a return code of 1, but its output shows the true error code so it is
-            # necessary to parse the output
-            return BlockedStatus(process_pbm_error(e.output))
         except Exception as e:
             # pbm pipes a return code of 1, but its output shows the true error code so it is
             # necessary to parse the output
@@ -652,3 +647,48 @@ def _get_backup_restore_operation_result(self, current_pbm_status, previous_pbm_
                 return f"Backup {backup_id} completed successfully"
 
         return "Unknown operation result"
+
+    def retrieve_error_message(self, pbm_status: Dict) -> str:
+        """Parses pbm status for an error message from the current unit.
+
+        If pbm_agent is in the error state, the command `pbm status` does not raise an error.
+        Instead, it is in the log messages. pbm_agent also shows all the error messages for other
+        replicas in the set.
+        """
+        try:
+            clusters = pbm_status["cluster"]
+            for cluster in clusters:
+                if cluster["rs"] == self.charm.app.name:
+                    break
+
+            for host_info in cluster["nodes"]:
+                replica_info = f"mongodb/{self.charm._unit_ip(self.charm.unit)}:27107"
+                if host_info["host"] == replica_info:
+                    break
+
+            return str(host_info["errors"])
+        except KeyError:
+            return ""
+
+    _StrOrBytes = Union[str, bytes]
+
+    def process_pbm_error(self, pbm_status: Optional[_StrOrBytes]) -> str:
+        """Returns errors found in PBM status."""
+        if type(pbm_status) == bytes:
+            pbm_status = pbm_status.decode("utf-8")
+
+        try:
+            error_message = self.retrieve_error_message(json.loads(pbm_status))
+        except json.decoder.JSONDecodeError:
+            # if pbm status doesn't return a parsable dictionary it is an error message
+            # represented as a string
+            error_message = pbm_status
+
+        message = None
+        if "status code: 403" in error_message:
+            message = "s3 credentials are incorrect."
+        elif "status code: 404" in error_message:
+            message = "s3 configurations are incompatible."
+        elif "status code: 301" in error_message:
+            message = "s3 configurations are incompatible."
+        return message