diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 0bb594e358..2624dd4d68 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -298,7 +298,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): from collections import namedtuple from datetime import datetime from enum import Enum -from typing import Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Union from ops import JujuVersion, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( @@ -320,7 +320,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 21 +LIBPATCH = 19 PYDEPS = ["ops>=2.0.0"] @@ -377,24 +377,11 @@ class SecretsIllegalUpdateError(SecretError): """Secrets aren't yet available for Juju version used.""" -def get_encoded_dict( +def get_encoded_field( relation: Relation, member: Union[Unit, Application], field: str -) -> Optional[Dict[str, str]]: +) -> Union[str, List[str], Dict[str, str]]: """Retrieve and decode an encoded field from relation data.""" - data = json.loads(relation.data[member].get(field, "{}")) - if isinstance(data, dict): - return data - logger.error("Unexpected datatype for %s instead of dict.", str(data)) - - -def get_encoded_list( - relation: Relation, member: Union[Unit, Application], field: str -) -> Optional[List[str]]: - """Retrieve and decode an encoded field from relation data.""" - data = json.loads(relation.data[member].get(field, "[]")) - if isinstance(data, list): - return data - logger.error("Unexpected datatype for %s instead of list.", str(data)) + return json.loads(relation.data[member].get(field, "{}")) def set_encoded_field( @@ -419,11 +406,16 @@ def diff(event: RelationChangedEvent, bucket: Union[Unit, Application]) -> Diff: keys from the event relation databag. """ # Retrieve the old data from the data key in the application relation databag. - old_data = get_encoded_dict(event.relation, bucket, "data") + old_data = get_encoded_field(event.relation, bucket, "data") if not old_data: old_data = {} + if not isinstance(old_data, dict): + # We should never get here, added to re-assure pyright + logger.error("Previous databag diff is of a wrong type.") + old_data = {} + # Retrieve the new data from the event relation databag. new_data = ( {key: value for key, value in event.relation.data[event.app].items() if key != "data"} @@ -531,14 +523,9 @@ def get_content(self) -> Dict[str, str]: def set_content(self, content: Dict[str, str]) -> None: """Setting cached secret content.""" - if not self.meta: - return - - if content: + if self.meta: self.meta.set_content(content) self._secret_content = content - else: - self.meta.remove_all_revisions() def get_info(self) -> Optional[SecretInfo]: """Wrapper function to apply the corresponding call on the Secret object within CachedSecret if any.""" @@ -635,16 +622,6 @@ def _fetch_my_specific_relation_data( """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" raise NotImplementedError - @abstractmethod - def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: - """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" - raise NotImplementedError - - @abstractmethod - def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: - """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" - raise NotImplementedError - # Internal helper methods @staticmethod @@ -711,9 +688,9 @@ def _group_secret_fields(secret_fields: List[str]) -> Dict[SecretGroup, List[str secret_fieldnames_grouped.setdefault(SecretGroup.EXTRA, []).append(key) return secret_fieldnames_grouped - def _get_group_secret_contents( + def _retrieve_group_secret_contents( self, - relation: Relation, + relation_id: int, group: SecretGroup, secret_fields: Optional[Union[Set[str], List[str]]] = None, ) -> Dict[str, str]: @@ -721,30 +698,12 @@ def _get_group_secret_contents( if not secret_fields: secret_fields = [] - if (secret := self._get_relation_secret(relation.id, group)) and ( + if (secret := self._get_relation_secret(relation_id, group)) and ( secret_data := secret.get_content() ): return {k: v for k, v in secret_data.items() if k in secret_fields} return {} - @staticmethod - def _content_for_secret_group( - content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup - ) -> Dict[str, str]: - """Select : pairs from input, that belong to this particular Secret group.""" - if group_mapping == SecretGroup.EXTRA: - return { - k: v - for k, v in content.items() - if k in secret_fields and k not in SECRET_LABEL_MAP.keys() - } - - return { - k: v - for k, v in content.items() - if k in secret_fields and SECRET_LABEL_MAP.get(k) == group_mapping - } - @juju_secrets_only def _get_relation_secret_data( self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None @@ -754,38 +713,6 @@ def _get_relation_secret_data( if secret: return secret.get_content() - # Core operations on Relation Fields manipulations (regardless whether the field is in the databag or in a secret) - # Internal functions to be called directly from transparent public interface functions (+closely related helpers) - - def _process_secret_fields( - self, - relation: Relation, - req_secret_fields: Optional[List[str]], - impacted_rel_fields: List[str], - operation: Callable, - second_chance_as_normal_field: bool = True, - *args, - **kwargs, - ) -> Tuple[Dict[str, str], Set[str]]: - """Isolate target secret fields of manipulation, and execute requested operation by Secret Group.""" - result = {} - normal_fields = set(impacted_rel_fields) - if req_secret_fields and self.secrets_enabled: - normal_fields = normal_fields - set(req_secret_fields) - secret_fields = set(impacted_rel_fields) - set(normal_fields) - - secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) - - for group in secret_fieldnames_grouped: - # operation() should return nothing when all goes well - if group_result := operation(relation, group, secret_fields, *args, **kwargs): - result.update(group_result) - elif second_chance_as_normal_field: - # If it wasn't found as a secret, let's give it a 2nd chance as "normal" field - # Needed when Juju3 Requires meets Juju2 Provider - normal_fields |= set(secret_fieldnames_grouped[group]) - return (result, normal_fields) - def _fetch_relation_data_without_secrets( self, app: Application, relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: @@ -816,49 +743,43 @@ def _fetch_relation_data_with_secrets( Provides side itself). """ result = {} - normal_fields = [] - - if not fields: - all_fields = list(relation.data[app].keys()) - normal_fields = [field for field in all_fields if not self._is_secret_field(field)] - # There must have been secrets there - if all_fields != normal_fields and req_secret_fields: - # So we assemble the full fields list (without 'secret-' fields) - fields = normal_fields + req_secret_fields + normal_fields = fields + if not normal_fields: + normal_fields = list(relation.data[app].keys()) - if fields: - result, normal_fields = self._process_secret_fields( - relation, req_secret_fields, fields, self._get_group_secret_contents - ) + if req_secret_fields and self.secrets_enabled: + if fields: + # Processing from what was requested + normal_fields = set(fields) - set(req_secret_fields) + secret_fields = set(fields) - set(normal_fields) + + secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) + + for group in secret_fieldnames_grouped: + if contents := self._retrieve_group_secret_contents( + relation.id, group, secret_fields + ): + result.update(contents) + else: + # If it wasn't found as a secret, let's give it a 2nd chance as "normal" field + normal_fields |= set(secret_fieldnames_grouped[group]) + else: + # Processing from what is given, i.e. retrieving all + normal_fields = [ + f for f in relation.data[app].keys() if not self._is_secret_field(f) + ] + secret_fields = [f for f in relation.data[app].keys() if self._is_secret_field(f)] + for group in SecretGroup: + result.update( + self._retrieve_group_secret_contents(relation.id, group, req_secret_fields) + ) # Processing "normal" fields. May include leftover from what we couldn't retrieve as a secret. - # (Typically when Juju3 Requires meets Juju2 Provides) - if normal_fields: - result.update( - self._fetch_relation_data_without_secrets(app, relation, list(normal_fields)) - ) + result.update({k: relation.data[app][k] for k in normal_fields if k in relation.data[app]}) return result - def _update_relation_data_without_secrets( - self, app: Application, relation: Relation, data: Dict[str, str] - ): - """Updating databag contents when no secrets are involved.""" - if any(self._is_secret_field(key) for key in data.keys()): - raise SecretsIllegalUpdateError("Can't update secret {key}.") - - if relation: - relation.data[app].update(data) - - def _delete_relation_data_without_secrets( - self, app: Application, relation: Relation, fields: List[str] - ) -> None: - """Remove databag fields 'fields' from Relation.""" - for field in fields: - relation.data[app].pop(field) - - # Public interface methods - # Handling Relation Fields seamlessly, regardless if in databag or a Juju Secret + # Public methods def get_relation(self, relation_name, relation_id) -> Relation: """Safe way of retrieving a relation.""" @@ -958,19 +879,12 @@ def fetch_my_relation_field( if relation_data := self.fetch_my_relation_data([relation_id], [field], relation_name): return relation_data.get(relation_id, {}).get(field) - @leader_only + # Public methods - mandatory override + + @abstractmethod def update_relation_data(self, relation_id: int, data: dict) -> None: """Update the data within the relation.""" - relation_name = self.relation_name - relation = self.get_relation(relation_name, relation_id) - return self._update_relation_data(relation, data) - - @leader_only - def delete_relation_data(self, relation_id: int, fields: List[str]) -> None: - """Remove field from the relation.""" - relation_name = self.relation_name - relation = self.get_relation(relation_name, relation_id) - return self._delete_relation_data(relation, fields) + raise NotImplementedError # Base DataProvides and DataRequires @@ -996,32 +910,36 @@ def _diff(self, event: RelationChangedEvent) -> Diff: # Private methods handling secrets + @leader_only @juju_secrets_only def _add_relation_secret( - self, relation: Relation, content: Dict[str, str], group_mapping: SecretGroup + self, relation_id: int, content: Dict[str, str], group_mapping: SecretGroup ) -> Optional[Secret]: """Add a new Juju Secret that will be registered in the relation databag.""" + relation = self.get_relation(self.relation_name, relation_id) + secret_field = self._generate_secret_field_name(group_mapping) if relation.data[self.local_app].get(secret_field): - logging.error("Secret for relation %s already exists, not adding again", relation.id) + logging.error("Secret for relation %s already exists, not adding again", relation_id) return - label = self._generate_secret_label(self.relation_name, relation.id, group_mapping) + label = self._generate_secret_label(self.relation_name, relation_id, group_mapping) secret = self.secrets.add(label, content, relation) # According to lint we may not have a Secret ID if secret.meta and secret.meta.id: relation.data[self.local_app][secret_field] = secret.meta.id + @leader_only @juju_secrets_only def _update_relation_secret( - self, relation: Relation, content: Dict[str, str], group_mapping: SecretGroup + self, relation_id: int, content: Dict[str, str], group_mapping: SecretGroup ): """Update the contents of an existing Juju Secret, referred in the relation databag.""" - secret = self._get_relation_secret(relation.id, group_mapping) + secret = self._get_relation_secret(relation_id, group_mapping) if not secret: - logging.error("Can't update secret for relation %s", relation.id) + logging.error("Can't update secret for relation %s", relation_id) return old_content = secret.get_content() @@ -1029,40 +947,22 @@ def _update_relation_secret( full_content.update(content) secret.set_content(full_content) - def _add_or_update_relation_secrets( - self, - relation: Relation, - group: SecretGroup, - secret_fields: Set[str], - data: Dict[str, str], - ) -> None: - """Update contents for Secret group. If the Secret doesn't exist, create it.""" - secret_content = self._content_for_secret_group(data, secret_fields, group) - if self._get_relation_secret(relation.id, group): - self._update_relation_secret(relation, secret_content, group) - else: - self._add_relation_secret(relation, secret_content, group) - - @juju_secrets_only - def _delete_relation_secret( - self, relation: Relation, group: SecretGroup, secret_fields: List[str], fields: List[str] - ): - """Update the contents of an existing Juju Secret, referred in the relation databag.""" - secret = self._get_relation_secret(relation.id, group) - - if not secret: - logging.error("Can't update secret for relation %s", relation.id) - return - - old_content = secret.get_content() - new_content = copy.deepcopy(old_content) - for field in fields: - new_content.pop(field) - secret.set_content(new_content) + @staticmethod + def _secret_content_grouped( + content: Dict[str, str], secret_fields: Set[str], group_mapping: SecretGroup + ) -> Dict[str, str]: + if group_mapping == SecretGroup.EXTRA: + return { + k: v + for k, v in content.items() + if k in secret_fields and k not in SECRET_LABEL_MAP.keys() + } - if not new_content: - field = self._generate_secret_field_name(group) - relation.data[self.local_app].pop(field) + return { + k: v + for k, v in content.items() + if k in secret_fields and SECRET_LABEL_MAP.get(k) == group_mapping + } # Mandatory internal overrides @@ -1104,42 +1004,45 @@ def _fetch_my_specific_relation_data( """Fetching our own relation data.""" secret_fields = None if relation.app: - secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + secret_fields = get_encoded_field(relation, relation.app, REQ_SECRET_FIELDS) return self._fetch_relation_data_with_secrets( self.local_app, - secret_fields, + secret_fields if isinstance(secret_fields, list) else None, relation, fields, ) - def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + # Public methods -- mandatory overrides + + @leader_only + def update_relation_data(self, relation_id: int, fields: Dict[str, str]) -> None: """Set values for fields not caring whether it's a secret or not.""" - req_secret_fields = [] - if relation.app: - req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + relation = self.get_relation(self.relation_name, relation_id) - _, normal_fields = self._process_secret_fields( - relation, - req_secret_fields, - list(data), - self._add_or_update_relation_secrets, - data=data, - ) + if relation.app: + relation_secret_fields = get_encoded_field(relation, relation.app, REQ_SECRET_FIELDS) + else: + relation_secret_fields = [] - normal_content = {k: v for k, v in data.items() if k in normal_fields} - self._update_relation_data_without_secrets(self.local_app, relation, normal_content) + normal_fields = list(fields) + if relation_secret_fields and self.secrets_enabled: + normal_fields = set(fields.keys()) - set(relation_secret_fields) + secret_fields = set(fields.keys()) - set(normal_fields) - def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: - """Delete fields from the Relation not caring whether it's a secret or not.""" - req_secret_fields = [] - if relation.app: - req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + secret_fieldnames_grouped = self._group_secret_fields(list(secret_fields)) - _, normal_fields = self._process_secret_fields( - relation, req_secret_fields, fields, self._delete_relation_secret, fields=fields + for group in secret_fieldnames_grouped: + secret_content = self._secret_content_grouped(fields, secret_fields, group) + if self._get_relation_secret(relation_id, group): + self._update_relation_secret(relation_id, secret_content, group) + else: + self._add_relation_secret(relation_id, secret_content, group) + + normal_content = {k: v for k, v in fields.items() if k in normal_fields} + relation.data[self.local_app].update( # pyright: ignore [reportGeneralTypeIssues] + normal_content ) - self._delete_relation_data_without_secrets(self.local_app, relation, list(normal_fields)) # Public methods - "native" @@ -1342,30 +1245,26 @@ def _fetch_my_specific_relation_data(self, relation, fields: Optional[List[str]] """Fetching our own relation data.""" return self._fetch_relation_data_without_secrets(self.local_app, relation, fields) - def _update_relation_data(self, relation: Relation, data: dict) -> None: + # Public methods -- mandatory overrides + + @leader_only + def update_relation_data(self, relation_id: int, data: dict) -> None: """Updates a set of key-value pairs in the relation. This function writes in the application data bag, therefore, only the leader unit can call it. Args: - relation: the particular relation. + relation_id: the identifier for a particular relation. data: dict containing the key-value pairs that should be updated in the relation. """ - return self._update_relation_data_without_secrets(self.local_app, relation, data) - - def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: - """Deletes a set of fields from the relation. - - This function writes in the application data bag, therefore, - only the leader unit can call it. + if any(self._is_secret_field(key) for key in data.keys()): + raise SecretsIllegalUpdateError("Requires side can't update secrets.") - Args: - relation: the particular relation. - fields: list containing the field names that should be removed from the relation. - """ - return self._delete_relation_data_without_secrets(self.local_app, relation, fields) + relation = self.charm.model.get_relation(self.relation_name, relation_id) + if relation: + relation.data[self.local_app].update(data) # General events @@ -1775,10 +1674,6 @@ def _assign_relation_alias(self, relation_id: int) -> None: if relation: relation.data[self.local_unit].update({"alias": available_aliases[0]}) - # We need to set relation alias also on the application level so, - # it will be accessible in show-unit juju command, executed for a consumer application unit - self.update_relation_data(relation_id, {"alias": available_aliases[0]}) - def _emit_aliased_event(self, event: RelationChangedEvent, event_name: str) -> None: """Emit an aliased event to a particular relation if it has an alias.