Dpe 5700 update lib so mongos-k8s removes users (#511)

## Issue
when the relation is broken between mongos-k8s and the config-server,
users in client relations are still accessible

## Solution
update the lib code so that mongos (when the relation to config-server
is broken):
1. removes these users + wipes their secrets
2. removes itself

lib/charms/mongodb/v0/config_server_interface.py

@@ -14,6 +14,7 @@
     DatabaseRequestedEvent,
     DatabaseRequires,
 )
+from charms.mongodb.v0.mongo import MongoConnection
 from charms.mongodb.v1.mongos import MongosConnection
 from ops.charm import (
     CharmBase,
@@ -30,6 +31,7 @@
     StatusBase,
     WaitingStatus,
 )
+from pymongo.errors import PyMongoError
 
 from config import Config
 
@@ -56,9 +58,13 @@ class ClusterProvider(Object):
     """Manage relations between the config server and mongos router on the config-server side."""
 
     def __init__(
-        self, charm: CharmBase, relation_name: str = Config.Relations.CLUSTER_RELATIONS_NAME
+        self,
+        charm: CharmBase,
+        relation_name: str = Config.Relations.CLUSTER_RELATIONS_NAME,
+        substrate: str = Config.Substrate.VM,
     ) -> None:
         """Constructor for ShardingProvider object."""
+        self.substrate = substrate
         self.relation_name = relation_name
         self.charm = charm
         self.database_provides = DatabaseProvides(self.charm, relation_name=self.relation_name)
@@ -185,7 +191,9 @@ def _on_relation_broken(self, event) -> None:
             logger.info("Skipping relation broken event, broken event due to scale down")
             return
 
-        self.charm.client_relations.oversee_users(departed_relation_id, event)
+        # mongos-k8s router is in charge of removing its own users.
+        if self.substrate == Config.Substrate.VM:
+            self.charm.client_relations.oversee_users(departed_relation_id, event)
 
     def update_config_server_db(self, event):
         """Provides related mongos applications with new config server db."""
@@ -314,6 +322,8 @@ def _on_relation_changed(self, event) -> None:
 
         # avoid restarting mongos when possible
         if not updated_keyfile and not updated_config and self.is_mongos_running():
+            # mongos-k8s router must update its users on start
+            self._update_k8s_users(event)
             return
 
         # mongos is not available until it is using new secrets
@@ -332,6 +342,20 @@ def _on_relation_changed(self, event) -> None:
         if self.charm.unit.is_leader():
             self.charm.mongos_initialised = True
 
+        # mongos-k8s router must update its users on start
+        self._update_k8s_users(event)
+
+    def _update_k8s_users(self, event) -> None:
+        if self.substrate != Config.Substrate.K8S:
+            return
+
+        # K8s can handle its 1:Many users after being initialized
+        try:
+            self.charm.client_relations.oversee_users(None, None)
+        except PyMongoError:
+            event.defer()
+            logger.debug("failed to add users on mongos-k8s router, will defer and try again.")
+
     def _on_relation_broken(self, event: RelationBrokenEvent) -> None:
         # Only relation_deparated events can check if scaling down
         if not self.charm.has_departed_run(event.relation.id):
@@ -345,6 +369,13 @@ def _on_relation_broken(self, event: RelationBrokenEvent) -> None:
             logger.info("Skipping relation broken event, broken event due to scale down")
             return
 
+        try:
+            self.handle_mongos_k8s_users_removal()
+        except PyMongoError:
+            logger.debug("Trouble removing router users, will defer and try again")
+            event.defer()
+            return
+
         self.charm.stop_mongos_service()
         logger.info("Stopped mongos daemon")
 
@@ -359,9 +390,24 @@ def _on_relation_broken(self, event: RelationBrokenEvent) -> None:
         if self.substrate == Config.Substrate.VM:
             self.charm.remove_connection_info()
         else:
-            self.db_initialised = False
+            self.charm.db_initialised = False
 
     # BEGIN: helper functions
+    def handle_mongos_k8s_users_removal(self) -> None:
+        """Handles the removal of all client mongos-k8s users and the mongos-k8s admin user.
+
+        Raises:
+            PyMongoError
+        """
+        if not self.charm.unit.is_leader() or self.substrate != Config.Substrate.K8S:
+            return
+
+        self.charm.client_relations.remove_all_relational_users()
+
+        # now that the client mongos users have been removed we can remove ourself
+        with MongoConnection(self.charm.mongo_config) as mongo:
+            mongo.drop_user(self.charm.mongo_config.username)
+
     def pass_hook_checks(self, event):
         """Runs the pre-hooks checks for ClusterRequirer, returns True if all pass."""
         if self.is_mongos_tls_missing():

lib/charms/mongodb/v1/mongodb_provider.py

@@ -227,6 +227,15 @@ def remove_users(
                 ):
                     continue
 
+                # for user removal of mongos-k8s router, we let the router remove itself
+                if (
+                    self.charm.is_role(Config.Role.CONFIG_SERVER)
+                    and self.substrate == Config.Substrate.K8S
+                ):
+                    logger.info("K8s routers will remove themselves.")
+                    self._remove_from_relational_users_to_manage(username)
+                    return
+
                 mongo.drop_user(username)
                 self._remove_from_relational_users_to_manage(username)
 
@@ -514,6 +523,34 @@ def _add_to_relational_users_to_manage(self, user_to_add: str) -> None:
         current_users.add(user_to_add)
         self._update_relational_users_to_manage(current_users)
 
+    def remove_all_relational_users(self):
+        """Removes all users from DB.
+
+        Raises: PyMongoError.
+        """
+        with MongoConnection(self.charm.mongo_config) as mongo:
+            database_users = mongo.get_users()
+
+        users_being_managed = database_users.intersection(self._get_relational_users_to_manage())
+        self.remove_users(users_being_managed, expected_current_users=set())
+
+        # now we must remove all of their connection info
+        for relation in self._get_relations():
+            fields = self.database_provides.fetch_my_relation_data([relation.id])[relation.id]
+            self.database_provides.delete_relation_data(relation.id, fields=list(fields))
+
+            # unforatunately the above doesn't work to remove secrets, so we forcibly remove the
+            # rest manually remove the secret before clearing the databag
+            for unit in relation.units:
+                secret_id = json.loads(relation.data[unit]["data"])["secret-user"]
+                # secret id is the same on all units for `secret-user`
+                break
+
+            user_secrets = self.charm.model.get_secret(id=secret_id)
+            user_secrets.remove_all_revisions()
+            user_secrets.get_content(refresh=True)
+            relation.data[self.charm.app].clear()
+
     @staticmethod
     def _get_database_from_relation(relation: Relation) -> Optional[str]:
         """Return database name from relation."""