From 079d0a347b102e80d6b72547b99ef57b7de444d7 Mon Sep 17 00:00:00 2001 From: Mia Altieri Date: Thu, 31 Aug 2023 08:20:19 +0000 Subject: [PATCH 1/6] sync libraries and update src as necessary --- lib/charms/mongodb/v0/helpers.py | 75 ++- lib/charms/mongodb/v0/mongodb_backups.py | 696 +++++++++++++--------- src/charm.py | 48 +- tests/unit/test_mongodb_backups.py | 729 ----------------------- 4 files changed, 549 insertions(+), 999 deletions(-) delete mode 100644 tests/unit/test_mongodb_backups.py diff --git a/lib/charms/mongodb/v0/helpers.py b/lib/charms/mongodb/v0/helpers.py index 1d7b81604..1dc877da7 100644 --- a/lib/charms/mongodb/v0/helpers.py +++ b/lib/charms/mongodb/v0/helpers.py @@ -1,16 +1,22 @@ """Simple functions, which can be used in both K8s and VM charms.""" # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. - import logging import os +import re import secrets import string import subprocess -from typing import List +from typing import List, Optional, Union from charms.mongodb.v0.mongodb import MongoDBConfiguration, MongoDBConnection -from ops.model import ActiveStatus, BlockedStatus, StatusBase, WaitingStatus +from ops.model import ( + ActiveStatus, + BlockedStatus, + MaintenanceStatus, + StatusBase, + WaitingStatus, +) from pymongo.errors import AutoReconnect, ServerSelectionTimeoutError # The unique Charmhub library identifier, never change it @@ -194,3 +200,66 @@ def copy_licenses_to_unit(): subprocess.check_output( "cp -r /snap/charmed-mongodb/current/licenses/* src/licenses", shell=True ) + + +_StrOrBytes = Union[str, bytes] + + +def process_pbm_error(error_string: Optional[_StrOrBytes]) -> str: + """Parses pbm error string and returns a user friendly message.""" + message = "couldn't configure s3 backup option" + if not error_string: + return message + if type(error_string) == bytes: + error_string = error_string.decode("utf-8") + if "status code: 403" in error_string: # type: ignore + message = "s3 credentials are incorrect." + elif "status code: 404" in error_string: # type: ignore + message = "s3 configurations are incompatible." + elif "status code: 301" in error_string: # type: ignore + message = "s3 configurations are incompatible." + return message + + +def current_pbm_op(pbm_status: str) -> str: + """Parses pbm status for the operation that pbm is running.""" + pbm_status_lines = pbm_status.splitlines() + for i in range(0, len(pbm_status_lines)): + line = pbm_status_lines[i] + + # operation is two lines after the line "Currently running:" + if line == "Currently running:": + return pbm_status_lines[i + 2] + + return "" + + +def process_pbm_status(pbm_status: str) -> StatusBase: + """Parses current pbm operation and returns unit status.""" + if type(pbm_status) == bytes: + pbm_status = pbm_status.decode("utf-8") + + # pbm is running resync operation + if "Resync" in current_pbm_op(pbm_status): + return WaitingStatus("waiting to sync s3 configurations.") + + # no operations are currently running with pbm + if "(none)" in current_pbm_op(pbm_status): + return ActiveStatus("") + + # Example of backup id: 2023-08-21T13:08:22Z + backup_match = re.search( + r'Snapshot backup "(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)"', pbm_status + ) + if backup_match: + backup_id = backup_match.group("backup_id") + return MaintenanceStatus(f"backup started/running, backup id:'{backup_id}'") + + restore_match = re.search( + r'Snapshot restore "(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)"', pbm_status + ) + if restore_match: + backup_id = restore_match.group("backup_id") + return MaintenanceStatus(f"restore started/running, backup id:'{backup_id}'") + + return ActiveStatus() diff --git a/lib/charms/mongodb/v0/mongodb_backups.py b/lib/charms/mongodb/v0/mongodb_backups.py index 6d7f59338..ed63496ab 100644 --- a/lib/charms/mongodb/v0/mongodb_backups.py +++ b/lib/charms/mongodb/v0/mongodb_backups.py @@ -3,27 +3,33 @@ """In this class, we manage backup configurations and actions. -Specifically backups are handled with Percona Backup MongoDB (pbm) which is installed as a snap -during the install phase. A user for PBM is created when MongoDB is first started during the -start phase. This user is named "backup". +Specifically backups are handled with Percona Backup MongoDB (pbm). +A user for PBM is created when MongoDB is first started during the start phase. +This user is named "backup". """ + import json import logging import re import subprocess import time -from typing import Dict +from typing import Dict, List from charms.data_platform_libs.v0.s3 import CredentialsChangedEvent, S3Requirer +from charms.mongodb.v0.helpers import ( + current_pbm_op, + process_pbm_error, + process_pbm_status, +) from charms.operator_libs_linux.v1 import snap from ops.framework import Object from ops.model import ( - ActiveStatus, BlockedStatus, MaintenanceStatus, StatusBase, WaitingStatus, ) +from ops.pebble import ExecError from tenacity import ( Retrying, before_log, @@ -34,15 +40,14 @@ ) # The unique Charmhub library identifier, never change it -LIBID = "18c461132b824ace91af0d7abe85f40e" +LIBID = "9f2b91c6128d48d6ba22724bf365da3b" # Increment this major API version when introducing breaking changes LIBAPI = 0 # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 5 -MONGODB_SNAP_DATA_DIR = "/var/snap/charmed-mongodb/current" +LIBPATCH = 1 logger = logging.getLogger(__name__) @@ -57,6 +62,10 @@ } S3_RELATION = "s3-credentials" REMAPPING_PATTERN = r"\ABackup doesn't match current cluster topology - it has different replica set names. Extra shards in the backup will cause this, for a simple example. The extra/unknown replica set names found in the backup are: ([^,\s]+)([.] Backup has no data for the config server or sole replicaset)?\Z" +PBM_STATUS_CMD = ["status", "-o", "json"] +MONGODB_SNAP_DATA_DIR = "/var/snap/charmed-mongodb/current" +BACKUP_RESTORE_MAX_ATTEMPTS = 5 +BACKUP_RESTORE_ATTEMPT_COOLDOWN = 15 class ResyncError(Exception): @@ -71,14 +80,39 @@ class PBMBusyError(Exception): """Raised when PBM is busy and cannot run another operation.""" +class RestoreError(Exception): + """Raised when backup operation is failed.""" + + +class BackupError(Exception): + """Raised when backup operation is failed.""" + + +def _backup_restore_retry_before_sleep(retry_state) -> None: + logger.error( + f"Attempt {retry_state.attempt_number} failed. {BACKUP_RESTORE_MAX_ATTEMPTS - retry_state.attempt_number} attempts left. Retrying after {BACKUP_RESTORE_ATTEMPT_COOLDOWN} seconds." + ), + + +def _backup_retry_stop_condition(retry_state) -> bool: + if isinstance(retry_state.outcome.exception(), BackupError): + return True + return retry_state.attempt_number >= BACKUP_RESTORE_MAX_ATTEMPTS + + +def _restore_retry_stop_condition(retry_state) -> bool: + if isinstance(retry_state.outcome.exception(), RestoreError): + return True + return retry_state.attempt_number >= BACKUP_RESTORE_MAX_ATTEMPTS + + class MongoDBBackups(Object): - """In this class, we manage mongodb backups.""" + """Manages MongoDB backups.""" - def __init__(self, charm, substrate="k8s"): + def __init__(self, charm): """Manager of MongoDB client relations.""" super().__init__(charm, "client-relations") self.charm = charm - self.substrate = substrate # s3 relation handles the config options for s3 backups self.s3_client = S3Requirer(self.charm, S3_RELATION) @@ -93,26 +127,178 @@ def _on_s3_credential_changed(self, event: CredentialsChangedEvent): """Sets pbm credentials, resyncs if necessary and reports config errors.""" # handling PBM configurations requires that MongoDB is running and the pbm snap is # installed. - if "db_initialised" not in self.charm.app_peer_data: - logger.debug("Cannot set PBM configurations, MongoDB has not yet started.") - event.defer() + action = "configure-pbm" + if not self.charm.db_initialised: + self._defer_action_with_info_log( + event, action, "Set PBM credentials, MongoDB not ready." + ) + return + + if not self.charm.has_backup_service(): + self._defer_action_with_info_log( + event, action, "Set PBM configurations, pbm-agent service not found." + ) + return + + self._configure_pbm_options(event) + + def _on_create_backup_action(self, event) -> None: + action = "backup" + if self.model.get_relation(S3_RELATION) is None: + self._fail_action_with_error_log( + event, + action, + "Relation with s3-integrator charm missing, cannot create backup.", + ) + return + + # only leader can create backups. This prevents multiple backups from being attempted at + # once. + if not self.charm.unit.is_leader(): + self._fail_action_with_error_log( + event, action, "The action can be run only on leader unit." + ) + return + + # cannot create backup if pbm is not ready. This could be due to: resyncing, incompatible, + # options, incorrect credentials, or already creating a backup + pbm_status = self._get_pbm_status() + self.charm.unit.status = pbm_status + + if isinstance(pbm_status, MaintenanceStatus): + self._fail_action_with_error_log( + event, + action, + "Can only create one backup at a time, please wait for current backup to finish.", + ) + return + + if isinstance(pbm_status, WaitingStatus): + self._defer_action_with_info_log( + event, + action, + "Sync-ing configurations needs more time, must wait before creating a backup.", + ) + return + + if isinstance(pbm_status, BlockedStatus): + self._fail_action_with_error_log(event, action, pbm_status.message) + return + + try: + backup_id = self._try_to_backup() + self.charm.unit.status = MaintenanceStatus( + f"backup started/running, backup id:'{backup_id}'" + ) + self._success_action_with_info_log( + event, action, {"backup-status": f"backup started. backup id: {backup_id}"} + ) + except (subprocess.CalledProcessError, ExecError, Exception) as e: + self._fail_action_with_error_log(event, action, str(e)) + return + + def _on_list_backups_action(self, event) -> None: + action = "list-backups" + if self.model.get_relation(S3_RELATION) is None: + self._fail_action_with_error_log( + event, + action, + "Relation with s3-integrator charm missing, cannot list backups.", + ) + return + + # cannot list backups if pbm is resyncing, or has incompatible options or incorrect + # credentials + pbm_status = self._get_pbm_status() + self.charm.unit.status = pbm_status + + if isinstance(pbm_status, WaitingStatus): + self._defer_action_with_info_log( + event, + action, + "Sync-ing configurations needs more time, must wait before listing backups.", + ) + return + + if isinstance(pbm_status, BlockedStatus): + self._fail_action_with_error_log(event, action, pbm_status.message) + return + + try: + formatted_list = self._generate_backup_list_output() + self._success_action_with_info_log(event, action, {"backups": formatted_list}) + except (subprocess.CalledProcessError, ExecError) as e: + self._fail_action_with_error_log(event, action, str(e)) + return + + def _on_restore_action(self, event) -> None: + action = "restore" + if self.model.get_relation(S3_RELATION) is None: + self._fail_action_with_error_log( + event, + action, + "Relation with s3-integrator charm missing, cannot restore from a backup.", + ) + return + + backup_id = event.params.get("backup-id") + if not backup_id: + self._fail_action_with_error_log(event, action, "Missing backup-id to restore") + return + + # only leader can restore backups. This prevents multiple restores from being attempted at + # once. + if not self.charm.unit.is_leader(): + self._fail_action_with_error_log( + event, action, "The action can be run only on leader unit." + ) + return + + # cannot restore backup if pbm is not ready. This could be due to: resyncing, incompatible, + # options, incorrect credentials, creating a backup, or already performing a restore. + pbm_status = self._get_pbm_status() + self.charm.unit.status = pbm_status + if isinstance(pbm_status, MaintenanceStatus): + self._fail_action_with_error_log( + event, action, "Please wait for current backup/restore to finish." + ) + return + + if isinstance(pbm_status, WaitingStatus): + self._defer_action_with_info_log( + event, + action, + "Sync-ing configurations needs more time, must wait before restoring.", + ) return - snap_cache = snap.SnapCache() - pbm_snap = snap_cache["charmed-mongodb"] - if not pbm_snap.present: - logger.debug("Cannot set PBM configurations, PBM snap is not yet installed.") - event.defer() + if isinstance(pbm_status, BlockedStatus): + self._fail_action_with_error_log( + event, action, f"Cannot restore backup {pbm_status.message}." + ) return - # pbm requires that the URI is set before adding configs - pbm_snap.set({"pbm-uri": self.charm.backup_config.uri}) + # sometimes when we are trying to restore pmb can be resyncing, so we need to retry + try: + self._try_to_restore(backup_id) + self.charm.unit.status = MaintenanceStatus( + f"restore started/running, backup id:'{backup_id}'" + ) + self._success_action_with_info_log( + event, action, {"restore-status": "restore started"} + ) + except ResyncError: + raise + except RestoreError as restore_error: + self._fail_action_with_error_log(event, action, str(restore_error)) + + # BEGIN: helper functions - # Add and sync configuration options while handling errors related to configuring options - # and re-syncing PBM. + def _configure_pbm_options(self, event) -> None: + action = "configure-pbm" try: - self._set_config_options(self._get_pbm_configs()) - self._resync_config_options(pbm_snap) + self._set_config_options() + self._resync_config_options() except SetPBMConfigError: self.charm.unit.status = BlockedStatus("couldn't configure s3 backup options.") return @@ -122,57 +308,58 @@ def _on_s3_credential_changed(self, event: CredentialsChangedEvent): return except ResyncError: self.charm.unit.status = WaitingStatus("waiting to sync s3 configurations.") - event.defer() - logger.debug("Sync-ing configurations needs more time.") + self._defer_action_with_info_log( + event, action, "Sync-ing configurations needs more time." + ) return except PBMBusyError: self.charm.unit.status = WaitingStatus("waiting to sync s3 configurations.") - logger.debug( - "Cannot update configs while PBM is running, must wait for PBM action to finish." - ) - event.defer() + self._defer_action_with_info_log( + event, + action, + "Cannot update configs while PBM is running, must wait for PBM action to finish.", + ), + return + except ExecError as e: + self.charm.unit.status = BlockedStatus(process_pbm_error(e.stdout)) return except subprocess.CalledProcessError as e: logger.error("Syncing configurations failed: %s", str(e)) self.charm.unit.status = self._get_pbm_status() - def _get_pbm_configs(self) -> Dict: - """Returns a dictionary of desired PBM configurations.""" - pbm_configs = {"storage.type": "s3"} - credentials = self.s3_client.get_s3_connection_info() - for s3_option, s3_value in credentials.items(): - if s3_option not in S3_PBM_OPTION_MAP: - continue - - pbm_configs[S3_PBM_OPTION_MAP[s3_option]] = s3_value - return pbm_configs - - def _set_config_options(self, pbm_configs): + def _set_config_options(self): """Applying given configurations with pbm.""" # clearing out configurations options before resetting them leads to a quicker reysnc # process - subprocess.check_output( - f"charmed-mongodb.pbm config --file {MONGODB_SNAP_DATA_DIR}/etc/pbm/pbm_config.yaml", - shell=True, - ) + self.charm.clear_pbm_config_file() # the pbm tool can only set one configuration at a time. - for pbm_key, pbm_value in pbm_configs.items(): + for pbm_key, pbm_value in self._get_pbm_configs().items(): try: - self._pbm_set_config(pbm_key, pbm_value) - except subprocess.CalledProcessError: - # do not log the error since the error outputs the command that was run. The - # command can include credentials that should not be logged. + config_cmd = ["config", "--set", f"{pbm_key}={pbm_value}"] + self.charm.run_pbm_command(config_cmd) + except (subprocess.CalledProcessError, ExecError): logger.error( "Failed to configure the PBM snap option: %s", pbm_key, ) raise SetPBMConfigError - def _resync_config_options(self, pbm_snap): + def _get_pbm_configs(self) -> Dict: + """Returns a dictionary of desired PBM configurations.""" + pbm_configs = {"storage.type": "s3"} + credentials = self.s3_client.get_s3_connection_info() + for s3_option, s3_value in credentials.items(): + if s3_option not in S3_PBM_OPTION_MAP: + continue + + pbm_configs[S3_PBM_OPTION_MAP[s3_option]] = s3_value + return pbm_configs + + def _resync_config_options(self): """Attempts to sync pbm config options and sets status in case of failure.""" - pbm_snap.start(services=["pbm-agent"]) + self.charm.start_backup_service() # pbm has a flakely resync and it is necessary to wait for no actions to be running before # resync-ing. See: https://jira.percona.com/browse/PBM-1038 @@ -189,50 +376,15 @@ def _resync_config_options(self, pbm_snap): # if a resync is running restart the service if isinstance(pbm_status, (WaitingStatus)): - pbm_snap.restart(services=["pbm-agent"]) + self.charm.restart_backup_service() raise PBMBusyError # wait for re-sync and update charm status based on pbm syncing status. Need to wait for # 2 seconds for pbm_agent to receive the resync command before verifying. - subprocess.check_output("charmed-mongodb.pbm config --force-resync", shell=True) + self.charm.run_pbm_command(["config", "--force-resync"]) time.sleep(2) self._wait_pbm_status() - def _get_pbm_status(self) -> StatusBase: - """Retrieve pbm status.""" - snap_cache = snap.SnapCache() - pbm_snap = snap_cache["charmed-mongodb"] - if not pbm_snap.present: - return BlockedStatus("pbm not installed.") - - try: - pbm_status = subprocess.check_output( - "charmed-mongodb.pbm status", shell=True, stderr=subprocess.STDOUT - ) - # pbm is running resync operation - if "Resync" in self._current_pbm_op(pbm_status.decode("utf-8")): - return WaitingStatus("waiting to sync s3 configurations.") - - # no operations are currently running with pbm - if "(none)" in self._current_pbm_op(pbm_status.decode("utf-8")): - return ActiveStatus("") - - if "Snapshot backup" in self._current_pbm_op(pbm_status.decode("utf-8")): - return MaintenanceStatus("backup started/running") - - if "Snapshot restore" in self._current_pbm_op(pbm_status.decode("utf-8")): - return MaintenanceStatus("restore started/running") - - except subprocess.CalledProcessError as e: - # pbm pipes a return code of 1, but its output shows the true error code so it is - # necessary to parse the output - error_message = e.output.decode("utf-8") - if "status code: 403" in error_message: - return BlockedStatus("s3 credentials are incorrect.") - - return BlockedStatus("s3 configurations are incompatible.") - return ActiveStatus("") - @retry( stop=stop_after_attempt(20), reraise=True, @@ -253,155 +405,177 @@ def _wait_pbm_status(self) -> None: """ # on occasion it takes the pbm_agent daemon time to update its configs, meaning that it # will error for incorrect configurations for <15s before resolving itself. + for attempt in Retrying( stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True, ): with attempt: - pbm_status = subprocess.check_output("charmed-mongodb.pbm status", shell=True) - if "Resync" in self._current_pbm_op(pbm_status.decode("utf-8")): - # since this process takes several minutes we should let the user know - # immediately. - self.charm.unit.status = WaitingStatus("waiting to sync s3 configurations.") - raise ResyncError - - def _current_pbm_op(self, pbm_status: str) -> str: - """Parses pbm status for the operation that pbm is running.""" - pbm_status = pbm_status.splitlines() - for i in range(0, len(pbm_status)): - line = pbm_status[i] - - # operation is two lines after the line "Currently running:" - if line == "Currently running:": - return pbm_status[i + 2] - - return "" - - def _pbm_set_config(self, key: str, value: str) -> None: - """Runs the charmed-mongodb.pbm config command for the provided key and value.""" - config_cmd = f'charmed-mongodb.pbm config --set {key}="{value}"' - subprocess.check_output(config_cmd, shell=True) + try: + pbm_status = self.charm.run_pbm_command(PBM_STATUS_CMD) + + if "Resync" in current_pbm_op(pbm_status): + # since this process takes several minutes we should let the user know + # immediately. + self.charm.unit.status = WaitingStatus( + "waiting to sync s3 configurations." + ) + raise ResyncError + except ExecError as e: + self.charm.unit.status = BlockedStatus(process_pbm_error(e.stdout)) - def _on_create_backup_action(self, event) -> None: - if self.model.get_relation(S3_RELATION) is None: - event.fail("Relation with s3-integrator charm missing, cannot create backup.") - return - - # only leader can create backups. This prevents multiple backups from being attempted at - # once. - if not self.charm.unit.is_leader(): - event.fail("The action can be run only on leader unit.") - return - - # cannot create backup if pbm is not ready. This could be due to: resyncing, incompatible, - # options, incorrect credentials, or already creating a backup - pbm_status = self._get_pbm_status() - self.charm.unit.status = pbm_status - if isinstance(pbm_status, MaintenanceStatus): - event.fail( - "Can only create one backup at a time, please wait for current backup to finish." - ) - return - if isinstance(pbm_status, WaitingStatus): - event.defer() - logger.debug( - "Sync-ing configurations needs more time, must wait before creating a backup." - ) - return - if isinstance(pbm_status, BlockedStatus): - event.fail(f"Cannot create backup {pbm_status.message}.") - return + def _get_pbm_status(self) -> StatusBase: + """Retrieve pbm status.""" + if not self.charm.has_backup_service(): + return WaitingStatus("waiting for pbm to start") try: - subprocess.check_output("charmed-mongodb.pbm backup", shell=True) - event.set_results({"backup-status": "backup started"}) - self.charm.unit.status = MaintenanceStatus("backup started/running") + previous_pbm_status = self.charm.unit.status + pbm_status = self.charm.run_pbm_command(PBM_STATUS_CMD) + self._log_backup_restore_result(pbm_status, previous_pbm_status) + return process_pbm_status(pbm_status) + except ExecError as e: + logger.error(f"Failed to get pbm status. {e}") + return BlockedStatus(process_pbm_error(e.stdout)) except subprocess.CalledProcessError as e: - event.fail(f"Failed to backup MongoDB with error: {str(e)}") - return - - def _on_list_backups_action(self, event) -> None: - if self.model.get_relation(S3_RELATION) is None: - event.fail("Relation with s3-integrator charm missing, cannot list backups.") - return + # pbm pipes a return code of 1, but its output shows the true error code so it is + # necessary to parse the output + return BlockedStatus(process_pbm_error(e.output)) + except Exception as e: + # pbm pipes a return code of 1, but its output shows the true error code so it is + # necessary to parse the output + logger.error(f"Failed to get pbm status: {e}") + return BlockedStatus("PBM error") - # cannot list backups if pbm is resyncing, or has incompatible options or incorrect - # credentials - pbm_status = self._get_pbm_status() - self.charm.unit.status = pbm_status - if isinstance(pbm_status, WaitingStatus): - event.defer() - logger.debug( - "Sync-ing configurations needs more time, must wait before listing backups." - ) - return - if isinstance(pbm_status, BlockedStatus): - event.fail(f"Cannot list backups: {pbm_status.message}.") - return + def _generate_backup_list_output(self) -> str: + """Generates a list of backups in a formatted table. - try: - formatted_list = self._generate_backup_list_output() - event.set_results({"backups": formatted_list}) - except subprocess.CalledProcessError as e: - event.fail(f"Failed to list MongoDB backups with error: {str(e)}") - return + List contains successful, failed, and in progress backups in order of ascending time. - def _on_restore_action(self, event) -> None: - if self.model.get_relation(S3_RELATION) is None: - event.fail("Relation with s3-integrator charm missing, cannot restore from a backup.") - return + Raises ExecError if pbm command fails. + """ + backup_list = [] + pbm_status = self.charm.run_pbm_command(["status", "--out=json"]) + # processes finished and failed backups + pbm_status = json.loads(pbm_status) + backups = pbm_status["backups"]["snapshot"] or [] + for backup in backups: + backup_status = "finished" + if backup["status"] == "error": + # backups from a different cluster have an error status, but they should show as + # finished + if self._backup_from_different_cluster(backup.get("error", "")): + backup_status = "finished" + else: + # display reason for failure if available + backup_status = "failed: " + backup.get("error", "N/A") + if backup["status"] not in ["error", "done"]: + backup_status = "in progress" + backup_list.append((backup["name"], backup["type"], backup_status)) - backup_id = event.params.get("backup-id") - if not backup_id: - event.fail("Missing backup-id to restore") - return + # process in progress backups + running_backup = pbm_status["running"] + if running_backup.get("type", None) == "backup": + # backups are sorted in reverse order + last_reported_backup = backup_list[0] + # pbm will occasionally report backups that are currently running as failed, so it is + # necessary to correct the backup list in this case. + if last_reported_backup[0] == running_backup["name"]: + backup_list[0] = (last_reported_backup[0], last_reported_backup[1], "in progress") + else: + backup_list.append((running_backup["name"], "logical", "in progress")) - # only leader can restore backups. This prevents multiple restores from being attempted at - # once. - if not self.charm.unit.is_leader(): - event.fail("The action can be run only on leader unit.") - return + # sort by time and return formatted output + return self._format_backup_list(sorted(backup_list, key=lambda pair: pair[0])) - # cannot restore backup if pbm is not ready. This could be due to: resyncing, incompatible, - # options, incorrect credentials, creating a backup, or already performing a restore. - pbm_status = self._get_pbm_status() - self.charm.unit.status = pbm_status - if isinstance(pbm_status, MaintenanceStatus): - event.fail("Please wait for current backup/restore to finish.") - return - if isinstance(pbm_status, WaitingStatus): - event.defer() - logger.debug("Sync-ing configurations needs more time, must wait before restoring.") - return - if isinstance(pbm_status, BlockedStatus): - event.fail(f"Cannot restore backup {pbm_status.message}.") - return + def _format_backup_list(self, backup_list: List[str]) -> str: + """Formats provided list of backups as a table.""" + backups = ["{:<21s} | {:<12s} | {:s}".format("backup-id", "backup-type", "backup-status")] - try: - remapping_args = self._remap_replicaset(backup_id) - subprocess.check_output( - f"charmed-mongodb.pbm restore {backup_id} {remapping_args}", - shell=True, - stderr=subprocess.STDOUT, + backups.append("-" * len(backups[0])) + for backup_id, backup_type, backup_status in backup_list: + backups.append( + "{:<21s} | {:<12s} | {:s}".format(backup_id, backup_type, backup_status) ) - event.set_results({"restore-status": "restore started"}) - self.charm.unit.status = MaintenanceStatus("restore started/running") - except subprocess.CalledProcessError as e: - error_message = e.output.decode("utf-8") - if f"backup '{backup_id}' not found" in error_message: - event.fail( - f"Backup id: {backup_id} does not exist in list of backups, please check list-backups for the available backup_ids." - ) - return - event.fail(f"Failed to restore MongoDB with error: {str(e)}") - return + return "\n".join(backups) def _backup_from_different_cluster(self, backup_status: str) -> bool: """Returns if a given backup was made on a different cluster.""" return re.search(REMAPPING_PATTERN, backup_status) is not None + def _try_to_restore(self, backup_id: str) -> None: + """Try to restore cluster a backup specified by backup id. + + If PBM is resyncing, the function will retry to create backup + (up to BACKUP_RESTORE_MAX_ATTEMPTS times) with BACKUP_RESTORE_ATTEMPT_COOLDOWN + time between attepts. + + If PMB returen any other error, the function will raise RestoreError. + """ + for attempt in Retrying( + stop=_restore_retry_stop_condition, + wait=wait_fixed(BACKUP_RESTORE_ATTEMPT_COOLDOWN), + reraise=True, + before_sleep=_backup_restore_retry_before_sleep, + ): + with attempt: + try: + remapping_args = self._remap_replicaset(backup_id) + restore_cmd = ["restore", backup_id] + if remapping_args: + restore_cmd = restore_cmd + remapping_args.split(" ") + self.charm.run_pbm_command(restore_cmd) + except (subprocess.CalledProcessError, ExecError) as e: + if type(e) == subprocess.CalledProcessError: + error_message = e.output.decode("utf-8") + else: + error_message = str(e.stderr) + if "Resync" in error_message: + raise ResyncError + + fail_message = f"Restore failed: {str(e)}" + if f"backup '{backup_id}' not found" in error_message: + fail_message = f"Restore failed: Backup id '{backup_id}' does not exist in list of backups, please check list-backups for the available backup_ids." + + raise RestoreError(fail_message) + + def _try_to_backup(self): + """Try to create a backup and return the backup id. + + If PBM is resyncing, the function will retry to create backup + (up to BACKUP_RESTORE_MAX_ATTEMPTS times) + with BACKUP_RESTORE_ATTEMPT_COOLDOWN time between attepts. + + If PMB returen any other error, the function will raise BackupError. + """ + for attempt in Retrying( + stop=_backup_retry_stop_condition, + wait=wait_fixed(BACKUP_RESTORE_ATTEMPT_COOLDOWN), + reraise=True, + before_sleep=_backup_restore_retry_before_sleep, + ): + with attempt: + try: + output = self.charm.run_pbm_command(["backup"]) + backup_id_match = re.search( + r"Starting backup '(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)'", + output, + ) + return backup_id_match.group("backup_id") if backup_id_match else "N/A" + except (subprocess.CalledProcessError, ExecError) as e: + if type(e) == subprocess.CalledProcessError: + error_message = e.output.decode("utf-8") + else: + error_message = str(e.stderr) + if "Resync" in error_message: + raise ResyncError + + fail_message = f"Backup failed: {str(e)}" + + raise BackupError(fail_message) + def _remap_replicaset(self, backup_id: str) -> str: """Returns options for remapping a replica set during a cluster migration restore. @@ -410,10 +584,8 @@ def _remap_replicaset(self, backup_id: str) -> str: Raises: CalledProcessError """ - pbm_status = subprocess.check_output( - "charmed-mongodb.pbm status --out=json", shell=True, stderr=subprocess.STDOUT - ) - pbm_status = json.loads(pbm_status.decode("utf-8")) + pbm_status = self.charm.run_pbm_command(PBM_STATUS_CMD) + pbm_status = json.loads(pbm_status) # grab the error status from the backup if present backups = pbm_status["backups"]["snapshot"] or [] @@ -439,57 +611,49 @@ def _remap_replicaset(self, backup_id: str) -> str: ) return f"--replset-remapping {current_cluster_name}={old_cluster_name}" - def _generate_backup_list_output(self) -> str: - """Generates a list of backups in a formatted table. + def _fail_action_with_error_log(self, event, action: str, message: str) -> None: + logger.error("%s failed: %s", action.capitalize(), message) + event.fail(message) - List contains successful, failed, and in progress backups in order of ascending time. + def _defer_action_with_info_log(self, event, action: str, message: str) -> None: + logger.info("Deferring %s: %s", action, message) + event.defer() + + def _success_action_with_info_log(self, event, action: str, results: Dict[str, str]) -> None: + logger.info("%s completed successfully", action.capitalize()) + event.set_results(results) + + def _log_backup_restore_result(self, current_pbm_status, previous_pbm_status) -> None: + """Logs the result of the backup/restore operation. - Raises CalledProcessError + Expected to be called for not failed operations. """ - backup_list = [] - pbm_status = subprocess.check_output( - "charmed-mongodb.pbm status --out=json", shell=True, stderr=subprocess.STDOUT + operation_result = self._get_backup_restore_operation_result( + current_pbm_status, previous_pbm_status ) - # processes finished and failed backups - pbm_status = json.loads(pbm_status.decode("utf-8")) - backups = pbm_status["backups"]["snapshot"] or [] - for backup in backups: - backup_status = "finished" - if backup["status"] == "error": - # backups from a different cluster have an error status, but they should show as - # finished - if self._backup_from_different_cluster(backup.get("error", "")): - backup_status = "finished" - else: - # display reason for failure if available - backup_status = "failed: " + backup.get("error", "N/A") - if backup["status"] not in ["error", "done"]: - backup_status = "in progress" - backup_list.append((backup["name"], backup["type"], backup_status)) + logger.info(operation_result) - # process in progress backups - running_backup = pbm_status["running"] - if running_backup.get("type", None) == "backup": - # backups are sorted in reverse order - last_reported_backup = backup_list[0] - # pbm will occasionally report backups that are currently running as failed, so it is - # necessary to correct the backup list in this case. - if last_reported_backup[0] == running_backup["name"]: - backup_list[0] = (last_reported_backup[0], last_reported_backup[1], "in progress") - else: - backup_list.append((running_backup["name"], "logical", "in progress")) + def _get_backup_restore_operation_result(self, current_pbm_status, previous_pbm_status) -> str: + """Returns a string with the result of the backup/restore operation. - # sort by time and return formatted output - return self._format_backup_list(sorted(backup_list, key=lambda pair: pair[0])) - - def _format_backup_list(self, backup_list) -> str: - """Formats provided list of backups as a table.""" - backups = ["{:<21s} | {:<12s} | {:s}".format("backup-id", "backup-type", "backup-status")] + The function call is expected to be only for not failed operations. + The operation is taken from previous status of the unit and expected + to contain the operation type (backup/restore) and the backup id. + """ + if ( + type(current_pbm_status) == type(previous_pbm_status) + and current_pbm_status.message == previous_pbm_status.message + ): + return f"Operation is still in progress: '{current_pbm_status.message}'" - backups.append("-" * len(backups[0])) - for backup_id, backup_type, backup_status in backup_list: - backups.append( - "{:<21s} | {:<12s} | {:s}".format(backup_id, backup_type, backup_status) - ) + if ( + type(previous_pbm_status) == MaintenanceStatus + and "backup id:" in previous_pbm_status.message + ): + backup_id = previous_pbm_status.message.split("backup id:")[-1].strip() + if "restore" in previous_pbm_status.message: + return f"Restore from backup {backup_id} completed successfully" + if "backup" in previous_pbm_status.message: + return f"Backup {backup_id} completed successfully" - return "\n".join(backups) + return "Unknown operation result" diff --git a/src/charm.py b/src/charm.py index 484d92e40..99f89d86a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -979,7 +979,53 @@ def auth_enabled(self) -> bool: return any("MONGOD_ARGS" in line and "--auth" in line for line in env_vars) - # END: helper functions + def has_backup_service(self): + """Verifies the backup service is available.""" + snap_cache = snap.SnapCache() + mongodb_snap = snap_cache["charmed-mongodb"] + if mongodb_snap.present: + return True + + return False + + def clear_pbm_config_file(self) -> None: + """Overwrites existing config file with the default file provided by snap.""" + subprocess.check_output( + f"charmed-mongodb.pbm config --file {Config.MONGODB_SNAP_DATA_DIR}/etc/pbm/pbm_config.yaml", + shell=True, + ) + + def run_pbm_command(self, cmd: List[str]) -> str: + """Executes the provided pbm command. + + Raises: + subprocess.CalledProcessError + """ + return subprocess.check_output(f"charmed-mongodb.pbm backup {' '.join(cmd)}", shell=True) + + def start_backup_service(self) -> None: + """Starts the pbm agent. + + Raises: + snap.SnapError + """ + snap_cache = snap.SnapCache() + charmed_mongodb_snap = snap_cache["charmed-mongodb"] + charmed_mongodb_snap.start(services=["pbm-agent"]) + + def restart_backup_service(self) -> None: + """Restarts the pbm agent. + + Raises: + snap.SnapError + """ + snap_cache = snap.SnapCache() + charmed_mongodb_snap = snap_cache["charmed-mongodb"] + charmed_mongodb_snap.restart(services=["pbm-agent"]) + + +# pbm_snap.restart(services=["pbm-agent"]) +# END: helper functions if __name__ == "__main__": diff --git a/tests/unit/test_mongodb_backups.py b/tests/unit/test_mongodb_backups.py deleted file mode 100644 index b54f59c83..000000000 --- a/tests/unit/test_mongodb_backups.py +++ /dev/null @@ -1,729 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -import unittest -from subprocess import CalledProcessError -from unittest import mock -from unittest.mock import patch - -import tenacity -from charms.mongodb.v0.mongodb_backups import ( - PBMBusyError, - ResyncError, - SetPBMConfigError, - stop_after_attempt, - wait_fixed, -) -from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus -from ops.testing import Harness - -from charm import MongodbOperatorCharm - -from .helpers import patch_network_get - -RELATION_NAME = "s3-credentials" - - -class TestMongoBackups(unittest.TestCase): - @patch_network_get(private_address="1.1.1.1") - def setUp(self): - self.harness = Harness(MongodbOperatorCharm) - self.harness.begin() - self.harness.add_relation("database-peers", "database-peers") - self.harness.set_leader(True) - self.charm = self.harness.charm - self.addCleanup(self.harness.cleanup) - - def test_current_pbm_op(self): - """Test if _current_pbm_op can identify the operation pbm is running.""" - action = self.harness.charm.backups._current_pbm_op( - "nothing\nCurrently running:\n====\nexpected action" - ) - self.assertEqual(action, "expected action") - - no_action = self.harness.charm.backups._current_pbm_op("pbm not started") - self.assertEqual(no_action, "") - - @patch("charm.snap.SnapCache") - def test_get_pbm_status_snap_not_present(self, snap): - """Tests that when the snap is not present pbm is in blocked state.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = False - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), BlockedStatus)) - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_get_pbm_status_resync(self, snap, output): - """Tests that when pbm is resyncing that pbm is in waiting state.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - output.return_value = b"Currently running:\n====\nResync op" - self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), WaitingStatus)) - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_get_pbm_status_running(self, snap, output): - """Tests that when pbm not running an op that pbm is in active state.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - output.return_value = b"Currently running:\n====\n(none)" - self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), ActiveStatus)) - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_get_pbm_status_backup(self, snap, output): - """Tests that when pbm running a backup that pbm is in maintenance state.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - output.return_value = b"Currently running:\n====\nSnapshot backup" - self.assertTrue( - isinstance(self.harness.charm.backups._get_pbm_status(), MaintenanceStatus) - ) - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_get_pbm_status_incorrect_cred(self, snap, output): - """Tests that when pbm has incorrect credentials that pbm is in blocked state.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), BlockedStatus)) - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_get_pbm_status_incorrect_conf(self, snap, output): - """Tests that when pbm has incorrect configs that pbm is in blocked state.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=42, output=b"" - ) - self.assertTrue(isinstance(self.harness.charm.backups._get_pbm_status(), BlockedStatus)) - - @patch("charm.subprocess.check_output") - @patch("charm.MongoDBBackups._get_pbm_status") - def test_verify_resync_config_error(self, _get_pbm_status, check_output): - """Tests that when pbm cannot perform the resync command it raises an error.""" - mock_snap = mock.Mock() - check_output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=42 - ) - - with self.assertRaises(CalledProcessError): - self.harness.charm.backups._resync_config_options(mock_snap) - - @patch("charm.subprocess.check_output") - def test_verify_resync_cred_error(self, check_output): - """Tests that when pbm cannot resync due to creds that it raises an error.""" - mock_snap = mock.Mock() - check_output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - with self.assertRaises(CalledProcessError): - self.harness.charm.backups._resync_config_options(mock_snap) - - @patch("charm.subprocess.check_output") - @patch("charm.MongoDBBackups._get_pbm_status") - def test_verify_resync_syncing(self, _get_pbm_status, check_output): - """Tests that when pbm needs more time to resync that it raises an error.""" - mock_snap = mock.Mock() - check_output.return_value = b"Currently running:\n====\nResync op" - - # disable retry - self.harness.charm.backups._wait_pbm_status.retry.retry = tenacity.retry_if_not_result( - lambda x: True - ) - - with self.assertRaises(ResyncError): - self.harness.charm.backups._resync_config_options(mock_snap) - - @patch("charms.mongodb.v0.mongodb_backups.wait_fixed") - @patch("charms.mongodb.v0.mongodb_backups.stop_after_attempt") - @patch("charm.MongoDBBackups._get_pbm_status") - def test_resync_config_options_failure(self, pbm_status, retry_stop, retry_wait): - """Verifies _resync_config_options raises an error when a resync cannot be performed.""" - pbm_status.return_value = MaintenanceStatus() - mock_snap = mock.Mock() - with self.assertRaises(PBMBusyError): - self.harness.charm.backups._resync_config_options(mock_snap) - - @patch("charm.subprocess.check_output") - @patch("charms.mongodb.v0.mongodb_backups.wait_fixed") - @patch("charms.mongodb.v0.mongodb_backups.stop_after_attempt") - @patch("charm.MongoDBBackups._get_pbm_status") - def test_resync_config_restart(self, pbm_status, retry_stop, retry_wait, check_output): - """Verifies _resync_config_options restarts that snap if alreaady resyncing.""" - retry_stop.return_value = stop_after_attempt(1) - retry_stop.return_value = wait_fixed(1) - pbm_status.return_value = WaitingStatus() - mock_snap = mock.Mock() - - with self.assertRaises(PBMBusyError): - self.harness.charm.backups._resync_config_options(mock_snap) - - mock_snap.restart.assert_called() - - @patch("charm.subprocess.check_output") - def test_set_config_options(self, check_output): - """Verifies _set_config_options failure raises SetPBMConfigError.""" - # the first check_output should succesd - check_output.side_effect = [ - None, - CalledProcessError( - cmd="charmed-mongodb.pbm config --set this_key=doesnt_exist", returncode=42 - ), - ] - - with self.assertRaises(SetPBMConfigError): - self.harness.charm.backups._set_config_options({"this_key": "doesnt_exist"}) - - def test_backup_without_rel(self): - """Verifies no backups are attempted without s3 relation.""" - action_event = mock.Mock() - action_event.params = {} - - self.harness.charm.backups._on_create_backup_action(action_event) - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_backup_syncing(self, snap, output): - """Verifies backup is deferred if more time is needed to resync.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - output.return_value = b"Currently running:\n====\nResync op" - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_create_backup_action(action_event) - - action_event.defer.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_backup_running_backup(self, snap, output): - """Verifies backup is fails if another backup is already running.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - output.return_value = b"Currently running:\n====\nSnapshot backup" - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_create_backup_action(action_event) - - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_backup_wrong_cred(self, snap, output): - """Verifies backup is fails if the credentials are incorrect.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_create_backup_action(action_event) - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.MongoDBBackups._get_pbm_status") - @patch("charm.snap.SnapCache") - def test_backup_failed(self, snap, pbm_status, output): - """Verifies backup is fails if the pbm command failed.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - pbm_status.return_value = ActiveStatus("") - - output.side_effect = CalledProcessError(cmd="charmed-mongodb.pbm backup", returncode=42) - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_create_backup_action(action_event) - - action_event.fail.assert_called() - - def test_backup_list_without_rel(self): - """Verifies no backup lists are attempted without s3 relation.""" - action_event = mock.Mock() - action_event.params = {} - - self.harness.charm.backups._on_list_backups_action(action_event) - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_backup_list_syncing(self, snap, output): - """Verifies backup list is deferred if more time is needed to resync.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - output.return_value = b"Currently running:\n====\nResync op" - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_list_backups_action(action_event) - - action_event.defer.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_backup_list_wrong_cred(self, snap, output): - """Verifies backup list fails with wrong credentials.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_list_backups_action(action_event) - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.MongoDBBackups._get_pbm_status") - @patch("charm.snap.SnapCache") - def test_backup_list_failed(self, snap, pbm_status, output): - """Verifies backup list fails if the pbm command fails.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {} - pbm_status.return_value = ActiveStatus("") - - output.side_effect = CalledProcessError(cmd="charmed-mongodb.pbm list", returncode=42) - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_list_backups_action(action_event) - - action_event.fail.assert_called() - - @patch("ops.framework.EventBase.defer") - def test_s3_credentials_no_db(self, defer): - """Verifies that when there is no DB that setting credentials is deferred.""" - del self.harness.charm.app_peer_data["db_initialised"] - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - defer.assert_called() - - @patch("ops.framework.EventBase.defer") - @patch("charm.snap.SnapCache") - def test_s3_credentials_no_snap(self, snap, defer): - """Verifies that when there is no DB that setting credentials is deferred.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = False - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - self.harness.charm.app_peer_data["db_initialised"] = "True" - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - defer.assert_called() - - @patch_network_get(private_address="1.1.1.1") - @patch("charm.snap.SnapCache") - @patch("charm.MongoDBBackups._set_config_options") - def test_s3_credentials_set_pbm_failure(self, _set_config_options, snap): - """Test charm goes into blocked state when setting pbm configs fail.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - mock_pbm_snap.set = mock.Mock() - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - _set_config_options.side_effect = SetPBMConfigError - self.harness.charm.app_peer_data["db_initialised"] = "True" - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) - - @patch_network_get(private_address="1.1.1.1") - @patch("charm.snap.SnapCache") - @patch("charm.MongoDBBackups._set_config_options") - @patch("charm.MongoDBBackups._resync_config_options") - @patch("ops.framework.EventBase.defer") - def test_s3_credentials_config_error(self, defer, resync, _set_config_options, snap): - """Test charm defers when more time is needed to sync pbm.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - mock_pbm_snap.set = mock.Mock() - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - self.harness.charm.app_peer_data["db_initialised"] = "True" - resync.side_effect = SetPBMConfigError - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) - - @patch_network_get(private_address="1.1.1.1") - @patch("charm.snap.SnapCache") - @patch("charm.MongoDBBackups._set_config_options") - @patch("charm.MongoDBBackups._resync_config_options") - @patch("ops.framework.EventBase.defer") - def test_s3_credentials_syncing(self, defer, resync, _set_config_options, snap): - """Test charm defers when more time is needed to sync pbm credentials.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - mock_pbm_snap.set = mock.Mock() - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - self.harness.charm.app_peer_data["db_initialised"] = "True" - resync.side_effect = ResyncError - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - defer.assert_called() - self.assertTrue(isinstance(self.harness.charm.unit.status, WaitingStatus)) - - @patch_network_get(private_address="1.1.1.1") - @patch("charm.snap.SnapCache") - @patch("charm.MongoDBBackups._set_config_options") - @patch("charm.MongoDBBackups._resync_config_options") - @patch("ops.framework.EventBase.defer") - def test_s3_credentials_pbm_busy(self, defer, resync, _set_config_options, snap): - """Test charm defers when more time is needed to sync pbm.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - mock_pbm_snap.set = mock.Mock() - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - self.harness.charm.app_peer_data["db_initialised"] = "True" - resync.side_effect = PBMBusyError - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - defer.assert_called() - self.assertTrue(isinstance(self.harness.charm.unit.status, WaitingStatus)) - - @patch_network_get(private_address="1.1.1.1") - @patch("charm.snap.SnapCache") - @patch("charm.MongoDBBackups._set_config_options") - @patch("charm.MongoDBBackups._resync_config_options") - @patch("ops.framework.EventBase.defer") - def test_s3_credentials_snap_start_error(self, defer, resync, _set_config_options, snap): - """Test charm defers when more time is needed to sync pbm.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - mock_pbm_snap.set = mock.Mock() - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - self.harness.charm.app_peer_data["db_initialised"] = "True" - resync.side_effect = snap.SnapError - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - defer.assert_not_called() - self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) - - @patch_network_get(private_address="1.1.1.1") - @patch("charm.snap.SnapCache") - @patch("charm.MongoDBBackups._set_config_options") - @patch("charm.MongoDBBackups._resync_config_options") - @patch("ops.framework.EventBase.defer") - @patch("charm.subprocess.check_output") - def test_s3_credentials_pbm_error(self, output, defer, resync, _set_config_options, snap): - """Test charm defers when more time is needed to sync pbm.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - mock_pbm_snap.set = mock.Mock() - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - self.harness.charm.app_peer_data["db_initialised"] = "True" - resync.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - - # triggering s3 event with correct fields - mock_s3_info = mock.Mock() - mock_s3_info.return_value = {"access-key": "noneya", "secret-key": "business"} - self.harness.charm.backups.s3_client.get_s3_connection_info = mock_s3_info - relation_id = self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.add_relation_unit(relation_id, "s3-integrator/0") - self.harness.update_relation_data( - relation_id, - "s3-integrator/0", - {"bucket": "hat"}, - ) - - defer.assert_not_called() - self.assertTrue(isinstance(self.harness.charm.unit.status, BlockedStatus)) - - @patch("charm.subprocess.check_output") - def test_generate_backup_list_output(self, check_output): - """Tests correct formation of backup list output. - - Specifically the spacing of the backups, the header, the backup order, and the backup - contents. - """ - # case 1: running backup is listed in error state - with open("tests/unit/data/pbm_status_duplicate_running.txt") as f: - output_contents = f.readlines() - output_contents = "".join(output_contents) - - check_output.return_value = output_contents.encode("utf-8") - formatted_output = self.harness.charm.backups._generate_backup_list_output() - formatted_output = formatted_output.split("\n") - header = formatted_output[0] - self.assertEqual(header, "backup-id | backup-type | backup-status") - divider = formatted_output[1] - self.assertEqual(divider, "-" * len(header)) - eariest_backup = formatted_output[2] - self.assertEqual( - eariest_backup, - "1900-02-14T13:59:14Z | physical | failed: internet not invented yet", - ) - failed_backup = formatted_output[3] - self.assertEqual(failed_backup, "2000-02-14T14:09:43Z | logical | finished") - inprogress_backup = formatted_output[4] - self.assertEqual(inprogress_backup, "2023-02-14T17:06:38Z | logical | in progress") - - # case 2: running backup is not listed in error state - with open("tests/unit/data/pbm_status.txt") as f: - output_contents = f.readlines() - output_contents = "".join(output_contents) - - check_output.return_value = output_contents.encode("utf-8") - formatted_output = self.harness.charm.backups._generate_backup_list_output() - formatted_output = formatted_output.split("\n") - header = formatted_output[0] - self.assertEqual(header, "backup-id | backup-type | backup-status") - divider = formatted_output[1] - self.assertEqual( - divider, "-" * len("backup-id | backup-type | backup-status") - ) - eariest_backup = formatted_output[2] - self.assertEqual( - eariest_backup, - "1900-02-14T13:59:14Z | physical | failed: internet not invented yet", - ) - failed_backup = formatted_output[3] - self.assertEqual(failed_backup, "2000-02-14T14:09:43Z | logical | finished") - inprogress_backup = formatted_output[4] - self.assertEqual(inprogress_backup, "2023-02-14T17:06:38Z | logical | in progress") - - def test_restore_without_rel(self): - """Verifies no restores are attempted without s3 relation.""" - action_event = mock.Mock() - action_event.params = {"backup-id": "back-me-up"} - - self.harness.charm.backups._on_restore_action(action_event) - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_restore_syncing(self, snap, output): - """Verifies restore is deferred if more time is needed to resync.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {"backup-id": "back-me-up"} - output.return_value = b"Currently running:\n====\nResync op" - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_restore_action(action_event) - - action_event.defer.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_restore_running_backup(self, snap, output): - """Verifies restore is fails if another backup is already running.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {"backup-id": "back-me-up"} - output.return_value = b"Currently running:\n====\nSnapshot backup" - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_restore_action(action_event) - - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.snap.SnapCache") - def test_restore_wrong_cred(self, snap, output): - """Verifies restore is fails if the credentials are incorrect.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {"backup-id": "back-me-up"} - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm status", returncode=403, output=b"status code: 403" - ) - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_restore_action(action_event) - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - @patch("charm.MongoDBBackups._get_pbm_status") - @patch("charm.snap.SnapCache") - def test_restore_failed(self, snap, pbm_status, output): - """Verifies restore is fails if the pbm command failed.""" - mock_pbm_snap = mock.Mock() - mock_pbm_snap.present = True - snap.return_value = {"charmed-mongodb": mock_pbm_snap} - - action_event = mock.Mock() - action_event.params = {"backup-id": "back-me-up"} - pbm_status.return_value = ActiveStatus("") - - output.side_effect = CalledProcessError( - cmd="charmed-mongodb.pbm backup", returncode=42, output=b"failed" - ) - - self.harness.add_relation(RELATION_NAME, "s3-integrator") - self.harness.charm.backups._on_restore_action(action_event) - - action_event.fail.assert_called() - - @patch("charm.subprocess.check_output") - def test_remap_replicaset_no_backup(self, check_output): - """Test verifies that no remapping is given if the backup_id doesn't exist.""" - with open("tests/unit/data/pbm_status.txt") as f: - output_contents = f.readlines() - output_contents = "".join(output_contents) - - check_output.return_value = output_contents.encode("utf-8") - remap = self.harness.charm.backups._remap_replicaset("this-id-doesnt-exist") - self.assertEqual(remap, "") - - @patch("charm.subprocess.check_output") - def test_remap_replicaset_no_remap_necessary(self, check_output): - """Test verifies that no remapping is given if no remapping is necessary.""" - with open("tests/unit/data/pbm_status_error_remap.txt") as f: - output_contents = f.readlines() - output_contents = "".join(output_contents) - - check_output.return_value = output_contents.encode("utf-8") - - # first case is that the backup is not in the error state - remap = self.harness.charm.backups._remap_replicaset("2000-02-14T14:09:43Z") - self.assertEqual(remap, "") - - # second case is that the backup has an error not related to remapping - remap = self.harness.charm.backups._remap_replicaset("1900-02-14T13:59:14Z") - self.assertEqual(remap, "") - - # third case is that the backup has two errors one related to remapping and another - # related to something else - remap = self.harness.charm.backups._remap_replicaset("2001-02-14T13:59:14Z") - self.assertEqual(remap, "") - - @patch("charm.subprocess.check_output") - def test_remap_replicaset_remap_necessary(self, check_output): - """Test verifies that remapping is provided and correct when necessary.""" - with open("tests/unit/data/pbm_status_error_remap.txt") as f: - output_contents = f.readlines() - output_contents = "".join(output_contents) - - check_output.return_value = output_contents.encode("utf-8") - self.harness.charm.app.name = "current-app-name" - - # first case is that the backup is not in the error state - remap = self.harness.charm.backups._remap_replicaset("2002-02-14T13:59:14Z") - self.assertEqual(remap, "--replset-remapping current-app-name=old-cluster-name") From 2fc61dddcf852b14987f0fc9c801acbf3c348efd Mon Sep 17 00:00:00 2001 From: Mia Altieri Date: Thu, 31 Aug 2023 08:29:00 +0000 Subject: [PATCH 2/6] fmt + lint --- lib/charms/mongodb/v0/mongodb_backups.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/charms/mongodb/v0/mongodb_backups.py b/lib/charms/mongodb/v0/mongodb_backups.py index ed63496ab..6319a1526 100644 --- a/lib/charms/mongodb/v0/mongodb_backups.py +++ b/lib/charms/mongodb/v0/mongodb_backups.py @@ -23,12 +23,7 @@ ) from charms.operator_libs_linux.v1 import snap from ops.framework import Object -from ops.model import ( - BlockedStatus, - MaintenanceStatus, - StatusBase, - WaitingStatus, -) +from ops.model import BlockedStatus, MaintenanceStatus, StatusBase, WaitingStatus from ops.pebble import ExecError from tenacity import ( Retrying, From b4fbfcec7fffe4e5da260a7d9a15285118014f1d Mon Sep 17 00:00:00 2001 From: Mia Altieri Date: Thu, 31 Aug 2023 10:16:42 +0000 Subject: [PATCH 3/6] update reqs --- requirements.txt | 4 ++-- src/charm.py | 26 ++++++++++++++------------ tox.ini | 16 ++++++++-------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/requirements.txt b/requirements.txt index 218fd9a4d..ff9e7f2be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ cosl==0.0.5 importlib-resources==5.10.2 tenacity==8.1.0 pymongo==4.3.3 -ops==2.0.0 +ops==2.4.1 jsonschema==4.17.3 cryptography==38.0.4 pure-sasl==0.6.2 @@ -12,7 +12,7 @@ pycparser==2.2 pkgutil-resolve-name==1.3.10 pydantic==1.10.7 pyrsistent==0.19.3 -pyyaml==6.0 +pyyaml==6.0.1 zipp==3.11.0 pyOpenSSL==22.1.0 typing-extensions==4.5.0 \ No newline at end of file diff --git a/src/charm.py b/src/charm.py index 99f89d86a..ae5ef0cd8 100755 --- a/src/charm.py +++ b/src/charm.py @@ -209,11 +209,13 @@ def _peers(self) -> Optional[Relation]: return self.model.get_relation(Config.Relations.PEERS) @property - def _db_initialised(self) -> bool: + def db_initialised(self) -> bool: + """Check if MongoDB is initialised.""" return "db_initialised" in self.app_peer_data - @_db_initialised.setter - def _db_initialised(self, value): + @db_initialised.setter + def db_initialised(self, value): + """Set the db_initialised flag.""" if isinstance(value, bool): self.app_peer_data["db_initialised"] = str(value) else: @@ -345,7 +347,7 @@ def _on_relation_handler(self, event: RelationEvent) -> None: # only leader should configure replica set and app-changed-events can trigger the relation # changed hook resulting in no JUJU_REMOTE_UNIT if this is the case we should return # further reconfiguration can be successful only if a replica set is initialised. - if not (self.unit.is_leader() and event.unit) or not self._db_initialised: + if not (self.unit.is_leader() and event.unit) or not self.db_initialised: return with MongoDBConnection(self.mongodb_config) as mongo: @@ -455,7 +457,7 @@ def _on_update_status(self, event: UpdateStatusEvent): return # no need to report on replica set status until initialised - if not self._db_initialised: + if not self.db_initialised: return # Cannot check more advanced MongoDB statuses if mongod hasn't started. @@ -676,7 +678,7 @@ def _generate_secrets(self) -> None: def _update_hosts(self, event: LeaderElectedEvent) -> None: """Update replica set hosts and remove any unremoved replicas from the config.""" - if not self._db_initialised: + if not self.db_initialised: return self.process_unremoved_units(event) @@ -684,7 +686,7 @@ def _update_hosts(self, event: LeaderElectedEvent) -> None: def update_app_relation_data(self) -> None: """Helper function to update application relation data.""" - if not self._db_initialised: + if not self.db_initialised: return database_users = set() @@ -826,7 +828,7 @@ def _push_tls_certificate_to_workload(self) -> None: def _connect_mongodb_exporter(self) -> None: """Exposes the endpoint to mongodb_exporter.""" - if not self._db_initialised: + if not self.db_initialised: return # must wait for leader to set URI before connecting @@ -840,7 +842,7 @@ def _connect_mongodb_exporter(self) -> None: def _connect_pbm_agent(self) -> None: """Updates URI for pbm-agent.""" - if not self._db_initialised: + if not self.db_initialised: return # must wait for leader to set URI before any attempts to update are made @@ -879,7 +881,7 @@ def _run_diagnostic_command(self, cmd) -> None: logger.error(f"Exception occurred running '{cmd}'\n {e}") def _initialise_replica_set(self, event: StartEvent) -> None: - if self._db_initialised: + if self.db_initialised: # The replica set should be initialised only once. Check should be # external (e.g., check initialisation inside peer relation). We # shouldn't rely on MongoDB response because the data directory @@ -913,7 +915,7 @@ def _initialise_replica_set(self, event: StartEvent) -> None: return # replica set initialised properly and ready to go - self._db_initialised = True + self.db_initialised = True self.unit.status = ActiveStatus() def _unit_ip(self, unit: Unit) -> str: @@ -1001,7 +1003,7 @@ def run_pbm_command(self, cmd: List[str]) -> str: Raises: subprocess.CalledProcessError """ - return subprocess.check_output(f"charmed-mongodb.pbm backup {' '.join(cmd)}", shell=True) + return subprocess.check_output(f"charmed-mongodb.pbm {' '.join(cmd)}", shell=True) def start_backup_service(self) -> None: """Starts the pbm agent. diff --git a/tox.ini b/tox.ini index 858f5402a..48c428ef7 100644 --- a/tox.ini +++ b/tox.ini @@ -72,7 +72,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -86,7 +86,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -100,7 +100,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -114,7 +114,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -128,7 +128,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -147,7 +147,7 @@ pass_env = GCP_SECRET_KEY deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -161,7 +161,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = @@ -176,7 +176,7 @@ pass_env = CI_PACKED_CHARMS deps = pytest - juju==2.9.38.1 # juju 3.3.0 has issues with retrieving action results + juju==2.9.44.0 # The latest python-libjuju that supports both juju 2.9 and 3.0 pytest-operator -r {tox_root}/requirements.txt commands = From 51e8557bc1bbbff9ae938a750d47d23672b82d7e Mon Sep 17 00:00:00 2001 From: Mia Altieri Date: Fri, 1 Sep 2023 09:50:36 +0000 Subject: [PATCH 4/6] process pbm response --- src/charm.py | 3 ++- tests/integration/backup_tests/test_backups.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index ae5ef0cd8..3b87b06be 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1003,7 +1003,8 @@ def run_pbm_command(self, cmd: List[str]) -> str: Raises: subprocess.CalledProcessError """ - return subprocess.check_output(f"charmed-mongodb.pbm {' '.join(cmd)}", shell=True) + pbm_response = subprocess.check_output(f"charmed-mongodb.pbm {' '.join(cmd)}", shell=True) + return pbm_response.decode("utf-8") def start_backup_service(self) -> None: """Starts the pbm agent. diff --git a/tests/integration/backup_tests/test_backups.py b/tests/integration/backup_tests/test_backups.py index 1436b047e..b7cb64d48 100644 --- a/tests/integration/backup_tests/test_backups.py +++ b/tests/integration/backup_tests/test_backups.py @@ -135,7 +135,7 @@ async def test_create_and_list_backups(ops_test: OpsTest) -> None: # verify backup is started action = await db_unit.run_action(action_name="create-backup") backup_result = await action.wait() - assert backup_result.results["backup-status"] == "backup started", "backup didn't start" + assert "backup started" in backup_result.results["backup-status"], "backup didn't start" # verify backup is present in the list of backups # the action `create-backup` only confirms that the command was sent to the `pbm`. Creating a @@ -401,4 +401,4 @@ async def test_update_backup_password(ops_test: OpsTest) -> None: # verify we still have connection to pbm via creating a backup action = await db_unit.run_action(action_name="create-backup") backup_result = await action.wait() - assert backup_result.results["backup-status"] == "backup started", "backup didn't start" + assert "backup started" in backup_result.results["backup-status"], "backup didn't start" From 64133ffc79d9a592e8317d77e525d9fe42d34826 Mon Sep 17 00:00:00 2001 From: Mia Altieri Date: Mon, 4 Sep 2023 13:32:43 +0000 Subject: [PATCH 5/6] resolve bug in status check --- lib/charms/mongodb/v0/helpers.py | 43 +++++++----------------- lib/charms/mongodb/v0/mongodb_backups.py | 2 +- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/lib/charms/mongodb/v0/helpers.py b/lib/charms/mongodb/v0/helpers.py index 1dc877da7..ba33c09d9 100644 --- a/lib/charms/mongodb/v0/helpers.py +++ b/lib/charms/mongodb/v0/helpers.py @@ -1,9 +1,9 @@ """Simple functions, which can be used in both K8s and VM charms.""" # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +import json import logging import os -import re import secrets import string import subprocess @@ -27,7 +27,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 7 +LIBPATCH = 8 # path to store mongodb ketFile @@ -223,43 +223,26 @@ def process_pbm_error(error_string: Optional[_StrOrBytes]) -> str: def current_pbm_op(pbm_status: str) -> str: """Parses pbm status for the operation that pbm is running.""" - pbm_status_lines = pbm_status.splitlines() - for i in range(0, len(pbm_status_lines)): - line = pbm_status_lines[i] - - # operation is two lines after the line "Currently running:" - if line == "Currently running:": - return pbm_status_lines[i + 2] - - return "" + pbm_status = json.loads(pbm_status) + return pbm_status["running"] if "running" in pbm_status else "" def process_pbm_status(pbm_status: str) -> StatusBase: """Parses current pbm operation and returns unit status.""" - if type(pbm_status) == bytes: - pbm_status = pbm_status.decode("utf-8") - - # pbm is running resync operation - if "Resync" in current_pbm_op(pbm_status): - return WaitingStatus("waiting to sync s3 configurations.") - + current_op = current_pbm_op(pbm_status) # no operations are currently running with pbm - if "(none)" in current_pbm_op(pbm_status): + if current_op == {}: return ActiveStatus("") - # Example of backup id: 2023-08-21T13:08:22Z - backup_match = re.search( - r'Snapshot backup "(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)"', pbm_status - ) - if backup_match: - backup_id = backup_match.group("backup_id") + if current_op["type"] == "backup": + backup_id = current_op["name"] return MaintenanceStatus(f"backup started/running, backup id:'{backup_id}'") - restore_match = re.search( - r'Snapshot restore "(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)"', pbm_status - ) - if restore_match: - backup_id = restore_match.group("backup_id") + if current_op["type"] == "restore": + backup_id = current_op["name"] return MaintenanceStatus(f"restore started/running, backup id:'{backup_id}'") + if current_op["type"] == "resync": + return WaitingStatus("waiting to sync s3 configurations.") + return ActiveStatus() diff --git a/lib/charms/mongodb/v0/mongodb_backups.py b/lib/charms/mongodb/v0/mongodb_backups.py index 6319a1526..6f705c4ba 100644 --- a/lib/charms/mongodb/v0/mongodb_backups.py +++ b/lib/charms/mongodb/v0/mongodb_backups.py @@ -42,7 +42,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 1 +LIBPATCH = 2 logger = logging.getLogger(__name__) From ff888497405632203e664e1cddedaae9e836e3c9 Mon Sep 17 00:00:00 2001 From: Mia Altieri Date: Tue, 5 Sep 2023 07:55:25 +0000 Subject: [PATCH 6/6] fix removal of cluster in tests --- lib/charms/mongodb/v0/mongodb_backups.py | 2 +- tests/integration/backup_tests/helpers.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/charms/mongodb/v0/mongodb_backups.py b/lib/charms/mongodb/v0/mongodb_backups.py index 6f705c4ba..c65c86f1d 100644 --- a/lib/charms/mongodb/v0/mongodb_backups.py +++ b/lib/charms/mongodb/v0/mongodb_backups.py @@ -42,7 +42,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 2 +LIBPATCH = 6 logger = logging.getLogger(__name__) diff --git a/tests/integration/backup_tests/helpers.py b/tests/integration/backup_tests/helpers.py index b4cb47682..a5ff7a633 100644 --- a/tests/integration/backup_tests/helpers.py +++ b/tests/integration/backup_tests/helpers.py @@ -1,6 +1,7 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. import os +import subprocess import ops from pymongo import MongoClient @@ -19,15 +20,20 @@ async def destroy_cluster(ops_test: OpsTest, cluster_name: str) -> None: # best practice to scale down before removing the entire cluster. Wait for cluster to settle # removing the next for i in range(0, len(units[:-1])): - await units[i].remove() + unit_name = units[i].name + await ops_test.model.applications[cluster_name].destroy_unit(unit_name) await ops_test.model.block_until( lambda: len(ops_test.model.applications[cluster_name].units) == len(units) - i - 1, timeout=TIMEOUT, ) - ops_test.model.wait_for_idle(apps=[cluster_name], status="active") + await ops_test.model.wait_for_idle(apps=[cluster_name], status="active") - # now that the cluster only has one unit left we can remove the application from Juju - await ops_test.model.applications[cluster_name].destroy() + # now that the cluster only has one unit left we can remove the application from Juju, send + # force for a quicker removal of the cluster. + model_name = ops_test.model.info.name + subprocess.check_output( + f"juju remove-application --model={model_name} --force new-mongodb".split() + ) # verify there are no more units. await ops_test.model.block_until(