From 67e1483161ddfe8473bb2542685b19f0e02c907a Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Thu, 2 Nov 2023 16:31:17 +0200 Subject: [PATCH 1/2] Conditional backup tests --- .github/workflows/ci.yaml | 8 +++++++- tests/integration/test_backups.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 93eea81c3f..8ea713372b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -123,8 +123,14 @@ jobs: echo Skipping unstable tests echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT" fi + - name: Select test secret usage + id: select-test-secrets + if: ${{ github.event.pull_request.head.repo.full_name != 'canonical/postgresql-operator' }} + run: | + echo Skipping tests using secrets + echo "mark_secrets=and not uses_secrets" >> "$GITHUB_OUTPUT" - name: Run integration tests - run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models + run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-secrets.outputs.mark_secrets }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models env: SECRETS_FROM_GITHUB: | { diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups.py index 07ee7a218a..e028b481dd 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups.py @@ -79,6 +79,12 @@ async def cloud_configs(ops_test: OpsTest, github_secrets) -> None: bucket_object.delete() +async def test_none() -> None: + """Empty test so that the suite will not fail if all tests are skippedi.""" + pass + + +@pytest.mark.uses_secrets @pytest.mark.abort_on_fail async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> None: """Build and deploy two units of PostgreSQL and then test the backup and restore actions.""" @@ -216,6 +222,7 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> No await ops_test.model.remove_application(TLS_CERTIFICATES_APP_NAME, block_until_done=True) +@pytest.mark.uses_secrets async def test_restore_on_new_cluster(ops_test: OpsTest) -> None: """Test that is possible to restore a backup to another PostgreSQL cluster.""" charm = await ops_test.build_charm(".") @@ -293,6 +300,7 @@ async def test_restore_on_new_cluster(ops_test: OpsTest) -> None: connection.close() +@pytest.mark.uses_secrets async def test_invalid_config_and_recovery_after_fixing_it( ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict] ) -> None: From add47f75607fe81dd67dda93ff951bc3c3ff0da2 Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Thu, 2 Nov 2023 17:18:00 +0200 Subject: [PATCH 2/2] Bump libs --- .../data_platform_libs/v0/data_interfaces.py | 34 +++++++++++++++---- lib/charms/postgresql_k8s/v0/postgresql.py | 4 +-- pyproject.toml | 3 ++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index df59585d53..5d1691d9f1 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -320,7 +320,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 22 +LIBPATCH = 23 PYDEPS = ["ops>=2.0.0"] @@ -807,6 +807,9 @@ def _fetch_relation_data_without_secrets( This is used typically when the Provides side wants to read the Requires side's data, or when the Requires side may want to read its own data. """ + if app not in relation.data or not relation.data[app]: + return {} + if fields: return {k: relation.data[app][k] for k in fields if k in relation.data[app]} else: @@ -830,6 +833,9 @@ def _fetch_relation_data_with_secrets( normal_fields = [] if not fields: + if app not in relation.data or not relation.data[app]: + return {} + all_fields = list(relation.data[app].keys()) normal_fields = [field for field in all_fields if not self._is_secret_field(field)] @@ -853,8 +859,11 @@ def _fetch_relation_data_with_secrets( def _update_relation_data_without_secrets( self, app: Application, relation: Relation, data: Dict[str, str] - ): + ) -> None: """Updating databag contents when no secrets are involved.""" + if app not in relation.data or relation.data[app] is None: + return + if any(self._is_secret_field(key) for key in data.keys()): raise SecretsIllegalUpdateError("Can't update secret {key}.") @@ -865,8 +874,19 @@ def _delete_relation_data_without_secrets( self, app: Application, relation: Relation, fields: List[str] ) -> None: """Remove databag fields 'fields' from Relation.""" + if app not in relation.data or not relation.data[app]: + return + for field in fields: - relation.data[app].pop(field) + try: + relation.data[app].pop(field) + except KeyError: + logger.debug( + "Non-existing field was attempted to be removed from the databag %s, %s", + str(relation.id), + str(field), + ) + pass # Public interface methods # Handling Relation Fields seamlessly, regardless if in databag or a Juju Secret @@ -880,9 +900,6 @@ def get_relation(self, relation_name, relation_id) -> Relation: "Relation %s %s couldn't be retrieved", relation_name, relation_id ) - if not relation.app: - raise DataInterfacesError("Relation's application missing") - return relation def fetch_relation_data( @@ -1089,7 +1106,10 @@ def _delete_relation_secret( # Remove secret from the relation if it's fully gone if not new_content: field = self._generate_secret_field_name(group) - relation.data[self.local_app].pop(field) + try: + relation.data[self.local_app].pop(field) + except KeyError: + pass # Return the content that was removed return True diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index debac9ae11..099047b01f 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -32,7 +32,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 18 +LIBPATCH = 19 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -514,7 +514,7 @@ def build_postgresql_parameters( ) if profile == "production": # Use 25% of the available memory for shared_buffers. - # and the remaind as cache memory. + # and the remaining as cache memory. shared_buffers = int(available_memory * 0.25) effective_cache_size = int(available_memory - shared_buffers) parameters.setdefault("shared_buffers", f"{int(shared_buffers/10**6)}MB") diff --git a/pyproject.toml b/pyproject.toml index 2831db7a3b..d73786fc79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,9 @@ target-version = ["py38"] # Linting tools configuration [tool.ruff] +# preview and explicit preview are enabled for CPY001 +preview = true +explicit-preview-rules = true target-version = "py38" src = ["src", "."] line-length = 99