From ce40968c98da939c13dc92dfc21f2c763ec6d4f1 Mon Sep 17 00:00:00 2001 From: Mykola Marzhan <303592+delgod@users.noreply.github.com> Date: Mon, 27 May 2024 19:19:14 +0200 Subject: [PATCH 1/3] [DPE-4416] Fetch charm libs to the latest LIBPATCH (update dp-libs to v36 to fix secrets issue) (#475) * [DPE-4416] Fetch charm libs to the latest LIBPATCH Co-authored-by: Alex Lutay <1928266+taurus-forever@users.noreply.github.com> --- .../data_platform_libs/v0/data_interfaces.py | 45 ++++++++++--------- lib/charms/data_platform_libs/v0/upgrade.py | 27 ++++++++--- lib/charms/operator_libs_linux/v2/snap.py | 9 ++-- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 3ce69e155e..b331bdce83 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -331,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 34 +LIBPATCH = 36 PYDEPS = ["ops>=2.0.0"] @@ -642,8 +642,8 @@ def _move_to_new_label_if_needed(self): return # Create a new secret with the new label - old_meta = self._secret_meta content = self._secret_meta.get_content() + self._secret_uri = None # I wish we could just check if we are the owners of the secret... try: @@ -651,7 +651,7 @@ def _move_to_new_label_if_needed(self): except ModelError as err: if "this unit is not the leader" not in str(err): raise - old_meta.remove_all_revisions() + self.current_label = None def set_content(self, content: Dict[str, str]) -> None: """Setting cached secret content.""" @@ -1586,7 +1586,7 @@ def _register_secret_to_relation( """ label = self._generate_secret_label(relation_name, relation_id, group) - # Fetchin the Secret's meta information ensuring that it's locally getting registered with + # Fetching the Secret's meta information ensuring that it's locally getting registered with CachedSecret(self._model, self.component, label, secret_id).meta def _register_secrets_to_relation(self, relation: Relation, params_name_list: List[str]): @@ -2309,7 +2309,7 @@ def _secrets(self) -> dict: return self._cached_secrets def _get_secret(self, group) -> Optional[Dict[str, str]]: - """Retrieveing secrets.""" + """Retrieving secrets.""" if not self.app: return if not self._secrets.get(group): @@ -3016,7 +3016,7 @@ class KafkaRequiresEvents(CharmEvents): # Kafka Provides and Requires -class KafkaProvidesData(ProviderData): +class KafkaProviderData(ProviderData): """Provider-side of the Kafka relation.""" def __init__(self, model: Model, relation_name: str) -> None: @@ -3059,12 +3059,12 @@ def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None: self.update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris}) -class KafkaProvidesEventHandlers(EventHandlers): +class KafkaProviderEventHandlers(EventHandlers): """Provider-side of the Kafka relation.""" on = KafkaProvidesEvents() # pyright: ignore [reportAssignmentType] - def __init__(self, charm: CharmBase, relation_data: KafkaProvidesData) -> None: + def __init__(self, charm: CharmBase, relation_data: KafkaProviderData) -> None: super().__init__(charm, relation_data) # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above self.relation_data = relation_data @@ -3086,15 +3086,15 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: ) -class KafkaProvides(KafkaProvidesData, KafkaProvidesEventHandlers): +class KafkaProvides(KafkaProviderData, KafkaProviderEventHandlers): """Provider-side of the Kafka relation.""" def __init__(self, charm: CharmBase, relation_name: str) -> None: - KafkaProvidesData.__init__(self, charm.model, relation_name) - KafkaProvidesEventHandlers.__init__(self, charm, self) + KafkaProviderData.__init__(self, charm.model, relation_name) + KafkaProviderEventHandlers.__init__(self, charm, self) -class KafkaRequiresData(RequirerData): +class KafkaRequirerData(RequirerData): """Requirer-side of the Kafka relation.""" def __init__( @@ -3124,12 +3124,12 @@ def topic(self, value): self._topic = value -class KafkaRequiresEventHandlers(RequirerEventHandlers): +class KafkaRequirerEventHandlers(RequirerEventHandlers): """Requires-side of the Kafka relation.""" on = KafkaRequiresEvents() # pyright: ignore [reportAssignmentType] - def __init__(self, charm: CharmBase, relation_data: KafkaRequiresData) -> None: + def __init__(self, charm: CharmBase, relation_data: KafkaRequirerData) -> None: super().__init__(charm, relation_data) # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above self.relation_data = relation_data @@ -3142,10 +3142,13 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: return # Sets topic, extra user roles, and "consumer-group-prefix" in the relation - relation_data = { - f: getattr(self, f.replace("-", "_"), "") - for f in ["consumer-group-prefix", "extra-user-roles", "topic"] - } + relation_data = {"topic": self.relation_data.topic} + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + + if self.relation_data.consumer_group_prefix: + relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix self.relation_data.update_relation_data(event.relation.id, relation_data) @@ -3188,7 +3191,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: return -class KafkaRequires(KafkaRequiresData, KafkaRequiresEventHandlers): +class KafkaRequires(KafkaRequirerData, KafkaRequirerEventHandlers): """Provider-side of the Kafka relation.""" def __init__( @@ -3200,7 +3203,7 @@ def __init__( consumer_group_prefix: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], ) -> None: - KafkaRequiresData.__init__( + KafkaRequirerData.__init__( self, charm.model, relation_name, @@ -3209,7 +3212,7 @@ def __init__( consumer_group_prefix, additional_secret_fields, ) - KafkaRequiresEventHandlers.__init__(self, charm, self) + KafkaRequirerEventHandlers.__init__(self, charm, self) # Opensearch related events diff --git a/lib/charms/data_platform_libs/v0/upgrade.py b/lib/charms/data_platform_libs/v0/upgrade.py index 4ee2e9ff88..ef74644de4 100644 --- a/lib/charms/data_platform_libs/v0/upgrade.py +++ b/lib/charms/data_platform_libs/v0/upgrade.py @@ -285,7 +285,7 @@ def restart(self, event) -> None: # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 15 +LIBPATCH = 16 PYDEPS = ["pydantic>=1.10,<2", "poetry-core"] @@ -501,7 +501,7 @@ class DataUpgrade(Object, ABC): STATES = ["recovery", "failed", "idle", "ready", "upgrading", "completed"] - on = UpgradeEvents() # pyright: ignore [reportGeneralTypeIssues] + on = UpgradeEvents() # pyright: ignore [reportAssignmentType] def __init__( self, @@ -606,6 +606,21 @@ def upgrade_stack(self, stack: List[int]) -> None: self.peer_relation.data[self.charm.app].update({"upgrade-stack": json.dumps(stack)}) self._upgrade_stack = stack + @property + def other_unit_states(self) -> list: + """Current upgrade state for other units. + + Returns: + Unsorted list of upgrade states for other units. + """ + if not self.peer_relation: + return [] + + return [ + self.peer_relation.data[unit].get("state", "") + for unit in list(self.peer_relation.units) + ] + @property def unit_states(self) -> list: """Current upgrade state for all units. @@ -926,11 +941,8 @@ def on_upgrade_changed(self, event: EventBase) -> None: return if self.substrate == "vm" and self.cluster_state == "recovery": - # Only defer for vm, that will set unit states to "ready" on upgrade-charm - # on k8s only the upgrading unit will receive the upgrade-charm event - # and deferring will prevent the upgrade stack from being popped - logger.debug("Cluster in recovery, deferring...") - event.defer() + # skip run while in recovery. The event will be retrigged when the cluster is ready + logger.debug("Cluster in recovery, skip...") return # if all units completed, mark as complete @@ -981,6 +993,7 @@ def on_upgrade_changed(self, event: EventBase) -> None: self.charm.unit == top_unit and top_state in ["ready", "upgrading"] and self.cluster_state == "ready" + and "upgrading" not in self.other_unit_states ): logger.debug( f"{top_unit.name} is next to upgrade, emitting `upgrade_granted` event and upgrading..." diff --git a/lib/charms/operator_libs_linux/v2/snap.py b/lib/charms/operator_libs_linux/v2/snap.py index ef426775d8..6d4dc385a6 100644 --- a/lib/charms/operator_libs_linux/v2/snap.py +++ b/lib/charms/operator_libs_linux/v2/snap.py @@ -83,7 +83,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 5 +LIBPATCH = 6 # Regex to locate 7-bit C1 ANSI sequences @@ -584,13 +584,16 @@ def ensure( "Installing snap %s, revision %s, tracking %s", self._name, revision, channel ) self._install(channel, cohort, revision) - else: + logger.info("The snap installation completed successfully") + elif revision is None or revision != self._revision: # The snap is installed, but we are changing it (e.g., switching channels). logger.info( "Refreshing snap %s, revision %s, tracking %s", self._name, revision, channel ) self._refresh(channel=channel, cohort=cohort, revision=revision, devmode=devmode) - logger.info("The snap installation completed successfully") + logger.info("The snap refresh completed successfully") + else: + logger.info("Refresh of snap %s was unnecessary", self._name) self._update_snap_apps() self._state = state From 1d35a6637e094ddaefd12a737ed41ee7deac5cda Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 29 May 2024 06:50:23 +0000 Subject: [PATCH 2/3] Run integration tests on arm64 (#464) --- .github/workflows/ci.yaml | 18 ++++++++----- .github/workflows/release.yaml | 4 +-- .github/workflows/sync_issue_to_jira.yaml | 2 +- poetry.lock | 25 +++++------------ pyproject.toml | 6 ++--- tests/integration/architecture.py | 7 +++++ .../ha_tests/test_async_replication.py | 7 ++++- .../ha_tests/test_upgrade_from_stable.py | 4 +++ tests/integration/markers.py | 7 +++++ .../new_relations/test_new_relations.py | 2 ++ tests/integration/test_backups.py | 27 ++++++++++++------- tests/integration/test_db.py | 2 ++ tests/integration/test_tls.py | 27 ++++++++++++------- 13 files changed, 86 insertions(+), 52 deletions(-) create mode 100644 tests/integration/architecture.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5e9c00f302..9910229afb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v13.1.2 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v13.2.0 unit-test: name: Unit test charm @@ -42,7 +42,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.1.2 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.2.0 with: cache: true @@ -53,21 +53,25 @@ jobs: juju: - agent: 2.9.49 # renovate: latest juju 2 libjuju: ==2.9.49.0 # renovate: latest libjuju 2 - allure: false + allure_on_amd64: false - agent: 3.1.8 # renovate: latest juju 3 - allure: true - name: Integration test charm | ${{ matrix.juju.agent }} + allure_on_amd64: true + architecture: + - amd64 + - arm64 + name: Integration test charm | ${{ matrix.juju.agent }} | ${{ matrix.architecture }} needs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v13.1.2 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v13.2.0 with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + architecture: ${{ matrix.architecture }} cloud: lxd juju-agent-version: ${{ matrix.juju.agent }} libjuju-version-constraint: ${{ matrix.juju.libjuju }} - _beta_allure_report: ${{ matrix.juju.allure }} + _beta_allure_report: ${{ matrix.juju.allure_on_amd64 && matrix.architecture == 'amd64' }} secrets: integration-test: | { diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 12fdddb6d5..ffc2119371 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,14 +24,14 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.1.2 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.2.0 release: name: Release charm needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v13.1.2 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v13.2.0 with: channel: 14/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} diff --git a/.github/workflows/sync_issue_to_jira.yaml b/.github/workflows/sync_issue_to_jira.yaml index 76bf6f7fd4..6fbe36ba41 100644 --- a/.github/workflows/sync_issue_to_jira.yaml +++ b/.github/workflows/sync_issue_to_jira.yaml @@ -9,7 +9,7 @@ on: jobs: sync: name: Sync GitHub issue to Jira - uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v13.1.2 + uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v13.2.0 with: jira-base-url: https://warthogs.atlassian.net jira-project-key: DPE diff --git a/poetry.lock b/poetry.lock index 0d36435368..a83bddad16 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1462,8 +1462,8 @@ develop = false [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v13.1.2" -resolved_reference = "f86cfdfbc92c929928c0722e7542867db0b092cd" +reference = "v13.2.0" +resolved_reference = "ba12595b3c9e91b245648a49c82dc181211a5769" subdirectory = "python/pytest_plugins/github_secrets" [[package]] @@ -1500,8 +1500,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v13.1.2" -resolved_reference = "f86cfdfbc92c929928c0722e7542867db0b092cd" +reference = "v13.2.0" +resolved_reference = "ba12595b3c9e91b245648a49c82dc181211a5769" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1519,8 +1519,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v13.1.2" -resolved_reference = "f86cfdfbc92c929928c0722e7542867db0b092cd" +reference = "v13.2.0" +resolved_reference = "ba12595b3c9e91b245648a49c82dc181211a5769" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -1560,7 +1560,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1568,16 +1567,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1594,7 +1585,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1602,7 +1592,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2062,4 +2051,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a613495dfd3f2d2c3ac0fdacd53e29a328cd49b45c63128b1f5faf6f34fcff17" +content-hash = "a73b8801ba490c66892f7cc917ab87480a8d29ae7cc37920023499f11536f6cd" diff --git a/pyproject.toml b/pyproject.toml index a5c7a4f866..ca1805e9de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,10 +66,10 @@ optional = true [tool.poetry.group.integration.dependencies] pytest = "^8.2.0" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.1.2", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.2.0", subdirectory = "python/pytest_plugins/github_secrets"} pytest-operator = "^0.35.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.1.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.1.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.2.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.2.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} # renovate caret doesn't work: https://github.com/renovatebot/renovate/issues/26940 juju = "<=3.4.0.0" boto3 = "*" diff --git a/tests/integration/architecture.py b/tests/integration/architecture.py new file mode 100644 index 0000000000..b0fbb34478 --- /dev/null +++ b/tests/integration/architecture.py @@ -0,0 +1,7 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +import subprocess + +architecture = subprocess.run( + ["dpkg", "--print-architecture"], capture_output=True, check=True, encoding="utf-8" +).stdout.strip() diff --git a/tests/integration/ha_tests/test_async_replication.py b/tests/integration/ha_tests/test_async_replication.py index 5f0c27dea8..bbb1991c7f 100644 --- a/tests/integration/ha_tests/test_async_replication.py +++ b/tests/integration/ha_tests/test_async_replication.py @@ -3,6 +3,7 @@ # See LICENSE file for licensing details. import contextlib import logging +import subprocess from asyncio import gather from typing import Optional @@ -12,7 +13,7 @@ from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_delay, wait_fixed -from .. import markers +from .. import architecture, markers from ..helpers import ( APPLICATION_NAME, DATABASE_APP_NAME, @@ -70,6 +71,10 @@ async def second_model(ops_test: OpsTest, first_model, request) -> Model: second_model_name = f"{first_model.info.name}-other" if second_model_name not in await ops_test._controller.list_models(): await ops_test._controller.add_model(second_model_name) + subprocess.run(["juju", "switch", second_model_name], check=True) + subprocess.run( + ["juju", "set-model-constraints", f"arch={architecture.architecture}"], check=True + ) second_model = Model() await second_model.connect(model_name=second_model_name) yield second_model diff --git a/tests/integration/ha_tests/test_upgrade_from_stable.py b/tests/integration/ha_tests/test_upgrade_from_stable.py index 515f1b701e..c1584c6a6c 100644 --- a/tests/integration/ha_tests/test_upgrade_from_stable.py +++ b/tests/integration/ha_tests/test_upgrade_from_stable.py @@ -6,6 +6,7 @@ import pytest from pytest_operator.plugin import OpsTest +from .. import markers from ..helpers import ( APPLICATION_NAME, DATABASE_APP_NAME, @@ -26,6 +27,7 @@ @pytest.mark.group(1) +@markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.abort_on_fail async def test_deploy_stable(ops_test: OpsTest) -> None: """Simple test to ensure that the PostgreSQL and application charms get deployed.""" @@ -77,6 +79,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.abort_on_fail async def test_pre_upgrade_check(ops_test: OpsTest) -> None: """Test that the pre-upgrade-check action runs successfully.""" @@ -95,6 +98,7 @@ async def test_pre_upgrade_check(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.abort_on_fail async def test_upgrade_from_stable(ops_test: OpsTest): """Test updating from stable channel.""" diff --git a/tests/integration/markers.py b/tests/integration/markers.py index 6dc2d6949f..2cfeab1c4f 100644 --- a/tests/integration/markers.py +++ b/tests/integration/markers.py @@ -4,8 +4,15 @@ import pytest +from . import architecture from .juju_ import juju_major_version juju2 = pytest.mark.skipif(juju_major_version != 2, reason="Requires juju 2") juju3 = pytest.mark.skipif(juju_major_version != 3, reason="Requires juju 3") juju_secrets = pytest.mark.skipif(juju_major_version < 3, reason="Requires juju secrets") +amd64_only = pytest.mark.skipif( + architecture.architecture != "amd64", reason="Requires amd64 architecture" +) +arm64_only = pytest.mark.skipif( + architecture.architecture != "arm64", reason="Requires arm64 architecture" +) diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations.py index 2c98f77bf9..779d50d573 100644 --- a/tests/integration/new_relations/test_new_relations.py +++ b/tests/integration/new_relations/test_new_relations.py @@ -12,6 +12,7 @@ import yaml from pytest_operator.plugin import OpsTest +from .. import markers from ..helpers import CHARM_SERIES, assert_sync_standbys, get_leader_unit, scale_application from ..juju_ import juju_major_version from .helpers import ( @@ -583,6 +584,7 @@ async def test_invalid_extra_user_roles(ops_test: OpsTest): @pytest.mark.group(1) +@markers.amd64_only # nextcloud charm not available for arm64 async def test_nextcloud_db_blocked(ops_test: OpsTest, charm: str) -> None: async with ops_test.fast_forward(): # Deploy Nextcloud. diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups.py index f63b714217..d5a473686f 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups.py @@ -10,6 +10,7 @@ from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_attempt, wait_exponential +from . import architecture from .helpers import ( CHARM_SERIES, DATABASE_APP_NAME, @@ -31,13 +32,19 @@ FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE = "failed to initialize stanza, check your S3 settings" S3_INTEGRATOR_APP_NAME = "s3-integrator" if juju_major_version < 3: - TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator" - TLS_CHANNEL = "legacy/stable" - TLS_CONFIG = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} + tls_certificates_app_name = "tls-certificates-operator" + if architecture.architecture == "arm64": + tls_channel = "legacy/edge" + else: + tls_channel = "legacy/stable" + tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} else: - TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" - TLS_CHANNEL = "latest/stable" - TLS_CONFIG = {"ca-common-name": "Test CA"} + tls_certificates_app_name = "self-signed-certificates" + if architecture.architecture == "arm64": + tls_channel = "latest/edge" + else: + tls_channel = "latest/stable" + tls_config = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) @@ -96,7 +103,7 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], charm """Build and deploy two units of PostgreSQL and then test the backup and restore actions.""" # Deploy S3 Integrator and TLS Certificates Operator. await ops_test.model.deploy(S3_INTEGRATOR_APP_NAME) - await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, config=TLS_CONFIG, channel=TLS_CHANNEL) + await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) for cloud, config in cloud_configs[0].items(): # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now @@ -111,7 +118,7 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], charm config={"profile": "testing"}, ) await ops_test.model.relate(database_app_name, S3_INTEGRATOR_APP_NAME) - await ops_test.model.relate(database_app_name, TLS_CERTIFICATES_APP_NAME) + await ops_test.model.relate(database_app_name, tls_certificates_app_name) # Configure and set access and secret keys. logger.info(f"configuring S3 integrator for {cloud}") @@ -223,7 +230,7 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], charm if cloud == list(cloud_configs[0].keys())[0]: # Remove the relation to the TLS certificates operator. await ops_test.model.applications[database_app_name].remove_relation( - f"{database_app_name}:certificates", f"{TLS_CERTIFICATES_APP_NAME}:certificates" + f"{database_app_name}:certificates", f"{tls_certificates_app_name}:certificates" ) await ops_test.model.wait_for_idle( apps=[database_app_name], status="active", timeout=1000 @@ -278,7 +285,7 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], charm await ops_test.model.remove_application(database_app_name, block_until_done=True) # Remove the TLS operator. - await ops_test.model.remove_application(TLS_CERTIFICATES_APP_NAME, block_until_done=True) + await ops_test.model.remove_application(tls_certificates_app_name, block_until_done=True) @pytest.mark.group(1) diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py index e9b67f9d8f..5ea134700f 100644 --- a/tests/integration/test_db.py +++ b/tests/integration/test_db.py @@ -193,6 +193,7 @@ async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest @pytest.mark.group(1) +@markers.amd64_only # sentry snap not available for arm64 async def test_sentry_db_blocked(ops_test: OpsTest, charm: str) -> None: async with ops_test.fast_forward(): # Deploy Sentry and its dependencies. @@ -332,6 +333,7 @@ async def test_roles_blocking(ops_test: OpsTest, charm: str) -> None: @markers.juju2 @pytest.mark.group(1) +@markers.amd64_only # canonical-livepatch-server charm (in bundle) not available for arm64 async def test_canonical_livepatch_onprem_bundle_db(ops_test: OpsTest) -> None: # Deploy and test the Livepatch onprem bundle (using this PostgreSQL charm # and an overlay to make the Ubuntu Advantage charm work with PostgreSQL). diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 70c7150470..d336f96259 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -8,6 +8,7 @@ from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_attempt, stop_after_delay, wait_exponential +from . import architecture from .helpers import ( CHARM_SERIES, DATABASE_APP_NAME, @@ -30,13 +31,19 @@ APP_NAME = METADATA["name"] if juju_major_version < 3: - TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator" - TLS_CHANNEL = "legacy/stable" - TLS_CONFIG = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} + tls_certificates_app_name = "tls-certificates-operator" + if architecture.architecture == "arm64": + tls_channel = "legacy/edge" + else: + tls_channel = "legacy/stable" + tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} else: - TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" - TLS_CHANNEL = "latest/stable" - TLS_CONFIG = {"ca-common-name": "Test CA"} + tls_certificates_app_name = "self-signed-certificates" + if architecture.architecture == "arm64": + tls_channel = "latest/edge" + else: + tls_channel = "latest/stable" + tls_config = {"ca-common-name": "Test CA"} @pytest.mark.group(1) @@ -63,11 +70,11 @@ async def test_tls_enabled(ops_test: OpsTest) -> None: async with ops_test.fast_forward(): # Deploy TLS Certificates operator. await ops_test.model.deploy( - TLS_CERTIFICATES_APP_NAME, config=TLS_CONFIG, channel=TLS_CHANNEL + tls_certificates_app_name, config=tls_config, channel=tls_channel ) # Relate it to the PostgreSQL to enable TLS. - await ops_test.model.relate(DATABASE_APP_NAME, TLS_CERTIFICATES_APP_NAME) + await ops_test.model.relate(DATABASE_APP_NAME, tls_certificates_app_name) await ops_test.model.wait_for_idle(status="active", timeout=1500) # Wait for all units enabling TLS. @@ -159,7 +166,7 @@ async def test_tls_enabled(ops_test: OpsTest) -> None: # Remove the relation. await ops_test.model.applications[DATABASE_APP_NAME].remove_relation( - f"{DATABASE_APP_NAME}:certificates", f"{TLS_CERTIFICATES_APP_NAME}:certificates" + f"{DATABASE_APP_NAME}:certificates", f"{tls_certificates_app_name}:certificates" ) await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1000) @@ -177,7 +184,7 @@ async def test_tls_enabled(ops_test: OpsTest) -> None: async def test_restart_machine(ops_test: OpsTest) -> None: async with ops_test.fast_forward(): # Relate it to the PostgreSQL to enable TLS. - await ops_test.model.relate(DATABASE_APP_NAME, TLS_CERTIFICATES_APP_NAME) + await ops_test.model.relate(DATABASE_APP_NAME, tls_certificates_app_name) await ops_test.model.wait_for_idle(status="active", timeout=1000) # Wait for all units enabling TLS. From 85f92e9a7121a1f506a550662c066343612a29e6 Mon Sep 17 00:00:00 2001 From: Marcelo Henrique Neppel Date: Fri, 31 May 2024 08:54:37 -0300 Subject: [PATCH 3/3] Fix scale up with S3 and TLS relations (#480) Signed-off-by: Marcelo Henrique Neppel --- src/backups.py | 21 +++++++++++++++ tests/unit/test_backups.py | 53 +++++++++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/backups.py b/src/backups.py index b1785b3a1b..d12e724d46 100644 --- a/src/backups.py +++ b/src/backups.py @@ -96,6 +96,22 @@ def _are_backup_settings_ok(self) -> Tuple[bool, Optional[str]]: return True, None + @property + def _can_initialise_stanza(self) -> bool: + """Validates whether this unit can initialise a stanza.""" + # Don't allow stanza initialisation if this unit hasn't started the database + # yet and either hasn't joined the peer relation yet or hasn't configured TLS + # yet while other unit already has TLS enabled. + if not self.charm._patroni.member_started and ( + (len(self.charm._peers.data.keys()) == 2) + or ( + "tls" not in self.charm.unit_peer_data + and any("tls" in unit_data for _, unit_data in self.charm._peers.data.items()) + ) + ): + return False + return True + def _can_unit_perform_backup(self) -> Tuple[bool, Optional[str]]: """Validates whether this unit can perform a backup.""" if self.charm.is_blocked: @@ -512,6 +528,11 @@ def _on_s3_credential_changed(self, event: CredentialsChangedEvent): logger.debug("Cannot set pgBackRest configurations, missing configurations.") return + if not self._can_initialise_stanza: + logger.debug("Cannot initialise stanza yet.") + event.defer() + return + # Verify the s3 relation only on the primary. if not self.charm.is_primary: return diff --git a/tests/unit/test_backups.py b/tests/unit/test_backups.py index 06229f7cdb..2e390e658f 100644 --- a/tests/unit/test_backups.py +++ b/tests/unit/test_backups.py @@ -73,6 +73,39 @@ def test_are_backup_settings_ok(harness): ) +def test_can_initialise_stanza(harness): + with patch("charm.Patroni.member_started", new_callable=PropertyMock) as _member_started: + # Test when Patroni or PostgreSQL hasn't started yet + # and the unit hasn't joined the peer relation yet. + _member_started.return_value = False + tc.assertEqual( + harness.charm.backup._can_initialise_stanza, + False, + ) + + # Test when the unit hasn't configured TLS yet while other unit already has TLS enabled. + harness.add_relation_unit( + harness.model.get_relation(PEER).id, f"{harness.charm.app.name}/1" + ) + with harness.hooks_disabled(): + harness.update_relation_data( + harness.model.get_relation(PEER).id, + f"{harness.charm.app.name}/1", + {"tls": "enabled"}, + ) + tc.assertEqual( + harness.charm.backup._can_initialise_stanza, + False, + ) + + # Test when everything is ok to initialise the stanza. + _member_started.return_value = True + tc.assertEqual( + harness.charm.backup._can_initialise_stanza, + True, + ) + + @patch_network_get(private_address="1.1.1.1") def test_can_unit_perform_backup(harness): with ( @@ -926,6 +959,9 @@ def test_on_s3_credential_changed(harness): patch( "charm.PostgresqlOperatorCharm.is_primary", new_callable=PropertyMock ) as _is_primary, + patch( + "charm.PostgreSQLBackups._can_initialise_stanza", new_callable=PropertyMock + ) as _can_initialise_stanza, patch( "charm.PostgreSQLBackups._render_pgbackrest_conf_file" ) as _render_pgbackrest_conf_file, @@ -953,31 +989,42 @@ def test_on_s3_credential_changed(harness): {"cluster_initialised": "True"}, ) _render_pgbackrest_conf_file.return_value = False - _is_primary.return_value = False harness.charm.backup.s3_client.on.credentials_changed.emit( relation=harness.model.get_relation(S3_PARAMETERS_RELATION, s3_rel_id) ) _defer.assert_not_called() _render_pgbackrest_conf_file.assert_called_once() + _can_initialise_stanza.assert_not_called() _create_bucket_if_not_exists.assert_not_called() _can_use_s3_repository.assert_not_called() _initialise_stanza.assert_not_called() + # Test when it's not possible to initialise the stanza in this unit. + _render_pgbackrest_conf_file.return_value = True + _can_initialise_stanza.return_value = False + harness.charm.backup.s3_client.on.credentials_changed.emit( + relation=harness.model.get_relation(S3_PARAMETERS_RELATION, s3_rel_id) + ) + _defer.assert_called_once() + _can_initialise_stanza.assert_called_once() + _is_primary.assert_not_called() + # Test that followers will not initialise the bucket harness.charm.unit.status = ActiveStatus() _render_pgbackrest_conf_file.reset_mock() + _can_initialise_stanza.return_value = True + _is_primary.return_value = False with harness.hooks_disabled(): harness.update_relation_data( peer_rel_id, harness.charm.app.name, {"cluster_initialised": "True"}, ) - _render_pgbackrest_conf_file.return_value = True - harness.charm.backup.s3_client.on.credentials_changed.emit( relation=harness.model.get_relation(S3_PARAMETERS_RELATION, s3_rel_id) ) _render_pgbackrest_conf_file.assert_called_once() + _is_primary.assert_called_once() _create_bucket_if_not_exists.assert_not_called() tc.assertIsInstance(harness.charm.unit.status, ActiveStatus) _can_use_s3_repository.assert_not_called()