From b18062dd3bc683556f9b8c0a44d10ab8c1d01cc9 Mon Sep 17 00:00:00 2001 From: shayancanonical <99665202+shayancanonical@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:59:56 -0400 Subject: [PATCH] [DPE-4976] Add workflow for nightly scheduled tests with juju 3.6 (#490) * Add temporary CI file to run juju 3.6 tests on branch PR * Explicitly use base instead of series, and hardcode base as ubuntu@22.04 * Add --base to a missed deploy command * Use channel 3/stable for kafka-k8s and zookeeper-k8s charms * Revert kafka and zookeeper to use latest/stable with base ubuntu@20.04 * Merge juju 3.6 workflow file into CI workflow file + upgrade to dpw v21.0.0 * Update outdated charm_tracing charm lib * Avoid setting revision for mongodb-k8s * Revert deployment of mongodb-k8s in mysql_root test * Use --base or --series based on juju version in test_mysql_root --- .github/workflows/ci.yaml | 13 +++++-- lib/charms/tempo_k8s/v1/charm_tracing.py | 17 +++++++-- poetry.lock | 23 ++++++------ pyproject.toml | 10 +++--- .../high_availability_helpers.py | 3 +- .../test_async_replication.py | 6 ++-- .../high_availability/test_upgrade.py | 2 ++ .../test_upgrade_from_stable.py | 2 ++ .../test_upgrade_rollback_incompat.py | 1 + tests/integration/relations/test_database.py | 3 +- .../integration/relations/test_mysql_root.py | 36 +++++++++++++------ tests/integration/test_backup_aws.py | 2 +- tests/integration/test_backup_ceph.py | 2 +- tests/integration/test_backup_gcp.py | 2 +- tests/integration/test_charm.py | 2 +- tests/integration/test_tls.py | 6 ++-- 16 files changed, 87 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 264cb774d..4432bb9b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -70,6 +70,8 @@ jobs: allure_on_amd64: false - agent: 3.4.5 # renovate: juju-agent-pin-minor allure_on_amd64: true + - snap_channel: 3.6/beta + allure_on_amd64: false architecture: - amd64 include: @@ -77,18 +79,23 @@ jobs: agent: 3.4.5 # renovate: juju-agent-pin-minor allure_on_amd64: true architecture: arm64 - name: Integration | ${{ matrix.juju.agent }} | ${{ matrix.architecture }} + - juju: + snap_channel: 3.6/beta + allure_on_amd64: false + architecture: arm64 + name: Integration | ${{ matrix.juju.agent || matrix.juju.snap_channel }} | ${{ matrix.architecture }} needs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v20.0.2 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v21.0.0 with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} architecture: ${{ matrix.architecture }} cloud: microk8s - microk8s-snap-channel: 1.28-strict/stable + microk8s-snap-channel: 1.28-strict/stable # renovate: latest microk8s juju-agent-version: ${{ matrix.juju.agent }} + juju-snap-channel: ${{ matrix.juju.snap_channel }} libjuju-version-constraint: ${{ matrix.juju.libjuju }} _beta_allure_report: ${{ matrix.juju.allure_on_amd64 && matrix.architecture == 'amd64' }} secrets: diff --git a/lib/charms/tempo_k8s/v1/charm_tracing.py b/lib/charms/tempo_k8s/v1/charm_tracing.py index fa926539b..2dbdddd68 100644 --- a/lib/charms/tempo_k8s/v1/charm_tracing.py +++ b/lib/charms/tempo_k8s/v1/charm_tracing.py @@ -224,7 +224,6 @@ def _remove_stale_otel_sdk_packages(): _remove_stale_otel_sdk_packages() - import functools import inspect import logging @@ -271,7 +270,7 @@ def _remove_stale_otel_sdk_packages(): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 14 +LIBPATCH = 15 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"] @@ -281,7 +280,6 @@ def _remove_stale_otel_sdk_packages(): # set this to 0 if you are debugging/developing this library source dev_logger.setLevel(logging.CRITICAL) - _CharmType = Type[CharmBase] # the type CharmBase and any subclass thereof _C = TypeVar("_C", bound=_CharmType) _T = TypeVar("_T", bound=type) @@ -333,9 +331,22 @@ def _get_tracer() -> Optional[Tracer]: try: return tracer.get() except LookupError: + # fallback: this course-corrects for a user error where charm_tracing symbols are imported + # from different paths (typically charms.tempo_k8s... and lib.charms.tempo_k8s...) try: ctx: Context = copy_context() if context_tracer := _get_tracer_from_context(ctx): + logger.warning( + "Tracer not found in `tracer` context var. " + "Verify that you're importing all `charm_tracing` symbols from the same module path. \n" + "For example, DO" + ": `from charms.lib...charm_tracing import foo, bar`. \n" + "DONT: \n" + " \t - `from charms.lib...charm_tracing import foo` \n" + " \t - `from lib...charm_tracing import bar` \n" + "For more info: https://python-notes.curiousefficiency.org/en/latest/python" + "_concepts/import_traps.html#the-double-import-trap" + ) return context_tracer.get() else: return None diff --git a/poetry.lock b/poetry.lock index 0a8de1ec1..238f47b19 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,8 +31,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v20.0.2" -resolved_reference = "37b15891a2ad9842721107fa4a42b6eb39ec830a" +reference = "v21.0.0" +resolved_reference = "c0eccd0a2229ce88cd09765d8260e22e12db0b13" subdirectory = "python/pytest_plugins/allure_pytest_collection_report" [[package]] @@ -896,12 +896,12 @@ referencing = ">=0.31.0" [[package]] name = "juju" -version = "3.5.0.0" +version = "3.5.2.0" description = "Python library for Juju" optional = false python-versions = "*" files = [ - {file = "juju-3.5.0.0.tar.gz", hash = "sha256:c69fbe63cb12991690787ce3d70812390bf3ca62b6c5e9ef15df00c1f03dd7e6"}, + {file = "juju-3.5.2.0.tar.gz", hash = "sha256:dd9a36330e63acd8f62bf478fd7e385e51f44dc3918e7a67d0593fd054e1e80a"}, ] [package.dependencies] @@ -1634,8 +1634,8 @@ develop = false [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v20.0.2" -resolved_reference = "37b15891a2ad9842721107fa4a42b6eb39ec830a" +reference = "v21.0.0" +resolved_reference = "c0eccd0a2229ce88cd09765d8260e22e12db0b13" subdirectory = "python/pytest_plugins/github_secrets" [[package]] @@ -1709,8 +1709,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v20.0.2" -resolved_reference = "37b15891a2ad9842721107fa4a42b6eb39ec830a" +reference = "v21.0.0" +resolved_reference = "c0eccd0a2229ce88cd09765d8260e22e12db0b13" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1728,8 +1728,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v20.0.2" -resolved_reference = "37b15891a2ad9842721107fa4a42b6eb39ec830a" +reference = "v21.0.0" +resolved_reference = "c0eccd0a2229ce88cd09765d8260e22e12db0b13" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -1782,7 +1782,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2389,4 +2388,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "c39b1a3dab6eb4ae7836d901affaad6c194c2fcb4c005787bfee4a4ff44079b5" +content-hash = "34993cff1c809487ca19de8147be5507649615e32c3ddeec9953a2fa2abf8ec5" diff --git a/pyproject.toml b/pyproject.toml index 8d2c7261c..fb5e8c41a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,12 +50,12 @@ parameterized = "^0.9.0" [tool.poetry.group.integration.dependencies] pytest = "^7.4.0" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v20.0.2", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/github_secrets"} pytest-microceph = {git = "https://github.com/canonical/data-platform-workflows", tag = "v20.0.2", subdirectory = "python/pytest_plugins/microceph"} pytest-operator = "^0.28.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v20.0.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v20.0.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"} -juju = "^3.2.2" +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +juju = "^3.5.2.0" ops = "^2.15.0" mysql-connector-python = "~8.0.33" tenacity = "^8.2.2" @@ -65,7 +65,7 @@ urllib3 = "^1.26.16" lightkube = "^0.14.0" kubernetes = "^27.2.0" allure-pytest = "^2.13.2" -allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v20.0.2", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} +allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} pytest-asyncio = "^0.21.1" [tool.coverage.run] diff --git a/tests/integration/high_availability/high_availability_helpers.py b/tests/integration/high_availability/high_availability_helpers.py index 6f138d6a6..e527682d6 100644 --- a/tests/integration/high_availability/high_availability_helpers.py +++ b/tests/integration/high_availability/high_availability_helpers.py @@ -160,7 +160,7 @@ async def deploy_and_scale_mysql( config=config, resources=resources, num_units=num_units, - series="jammy", + base="ubuntu@22.04", trust=True, ) @@ -198,6 +198,7 @@ async def deploy_and_scale_application(ops_test: OpsTest) -> str: application_name=APPLICATION_DEFAULT_APP_NAME, num_units=1, channel="latest/edge", + base="ubuntu@22.04", ) await ops_test.model.wait_for_idle( diff --git a/tests/integration/high_availability/test_async_replication.py b/tests/integration/high_availability/test_async_replication.py index 40a9e1a71..0a0c29b91 100644 --- a/tests/integration/high_availability/test_async_replication.py +++ b/tests/integration/high_availability/test_async_replication.py @@ -90,6 +90,7 @@ async def test_build_and_deploy( config=config, resources=resources, trust=True, + base="ubuntu@22.04", ) config["cluster-name"] = "cuzco" await second_model.deploy( @@ -99,6 +100,7 @@ async def test_build_and_deploy( config=config, resources=resources, trust=True, + base="ubuntu@22.04", ) logger.info("Waiting for the applications to settle") @@ -193,7 +195,7 @@ async def test_deploy_router_and_app(first_model: Model) -> None: await first_model.deploy( MYSQL_ROUTER_APP_NAME, application_name=MYSQL_ROUTER_APP_NAME, - series="jammy", + base="ubuntu@22.04", channel="8.0/edge", num_units=1, trust=True, @@ -201,7 +203,7 @@ async def test_deploy_router_and_app(first_model: Model) -> None: await first_model.deploy( APPLICATION_APP_NAME, application_name=APPLICATION_APP_NAME, - series="jammy", + base="ubuntu@22.04", channel="latest/edge", num_units=1, ) diff --git a/tests/integration/high_availability/test_upgrade.py b/tests/integration/high_availability/test_upgrade.py index 68f7a8f63..b71c6ad08 100644 --- a/tests/integration/high_availability/test_upgrade.py +++ b/tests/integration/high_availability/test_upgrade.py @@ -46,12 +46,14 @@ async def test_deploy_latest(ops_test: OpsTest) -> None: channel="8.0/edge", trust=True, config={"profile": "testing"}, + base="ubuntu@22.04", ) await ops_test.model.deploy( TEST_APP_NAME, application_name=TEST_APP_NAME, num_units=1, channel="latest/edge", + base="ubuntu@22.04", ) await relate_mysql_and_application(ops_test, MYSQL_APP_NAME, TEST_APP_NAME) diff --git a/tests/integration/high_availability/test_upgrade_from_stable.py b/tests/integration/high_availability/test_upgrade_from_stable.py index cfd30ece2..74ef62a6f 100644 --- a/tests/integration/high_availability/test_upgrade_from_stable.py +++ b/tests/integration/high_availability/test_upgrade_from_stable.py @@ -43,12 +43,14 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: channel="8.0/stable", trust=True, config={"profile": "testing"}, + base="ubuntu@22.04", ), ops_test.model.deploy( f"mysql-{TEST_APP_NAME}", application_name=TEST_APP_NAME, num_units=1, channel="latest/edge", + base="ubuntu@22.04", ), ) await relate_mysql_and_application(ops_test, MYSQL_APP_NAME, TEST_APP_NAME) diff --git a/tests/integration/high_availability/test_upgrade_rollback_incompat.py b/tests/integration/high_availability/test_upgrade_rollback_incompat.py index 1493e0bba..c316d4ba2 100644 --- a/tests/integration/high_availability/test_upgrade_rollback_incompat.py +++ b/tests/integration/high_availability/test_upgrade_rollback_incompat.py @@ -47,6 +47,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: num_units=3, resources=resources, trust=True, + base="ubuntu@22.04", ) await ops_test.model.wait_for_idle( diff --git a/tests/integration/relations/test_database.py b/tests/integration/relations/test_database.py index f24694440..64e228e22 100644 --- a/tests/integration/relations/test_database.py +++ b/tests/integration/relations/test_database.py @@ -44,7 +44,7 @@ async def test_build_and_deploy(ops_test: OpsTest): config=config, num_units=3, resources=resources, - series="jammy", + base="ubuntu@22.04", trust=True, ), ops_test.model.deploy( @@ -52,6 +52,7 @@ async def test_build_and_deploy(ops_test: OpsTest): application_name=APPLICATION_APP_NAME, num_units=2, channel="latest/edge", + base="ubuntu@22.04", ), ) diff --git a/tests/integration/relations/test_mysql_root.py b/tests/integration/relations/test_mysql_root.py index 27c478c10..486bbff5d 100644 --- a/tests/integration/relations/test_mysql_root.py +++ b/tests/integration/relations/test_mysql_root.py @@ -18,6 +18,7 @@ get_unit_address, is_relation_joined, ) +from ..juju_ import juju_major_version logger = logging.getLogger(__name__) @@ -44,6 +45,19 @@ async def test_deploy_and_relate_osm_bundle(ops_test: OpsTest) -> None: "image": "opensourcemano/pol:testing-daily", } + osm_keystone_deploy_commands = [ + "deploy", + "--channel=latest/beta", + "--resource", + "keystone-image=opensourcemano/keystone:testing-daily", + "osm-keystone", + ] + + if juju_major_version >= 3: + osm_keystone_deploy_commands.extend(["--base", "ubuntu@22.04"]) + else: + osm_keystone_deploy_commands.extend(["--series", "jammy"]) + await asyncio.gather( ops_test.model.deploy( charm, @@ -51,31 +65,33 @@ async def test_deploy_and_relate_osm_bundle(ops_test: OpsTest) -> None: resources=resources, config=config, num_units=1, - series="jammy", + base="ubuntu@22.04", trust=True, ), # Deploy the osm-keystone charm # (using ops_test.juju instead of ops_test.deploy as the latter does # not correctly deploy with the correct resources) - ops_test.juju( - "deploy", - "--channel=latest/beta", - "--resource", - "keystone-image=opensourcemano/keystone:testing-daily", - "osm-keystone", - ), + ops_test.juju(*osm_keystone_deploy_commands), ops_test.model.deploy( "osm-pol", application_name="osm-pol", channel="latest/beta", resources=osm_pol_resources, trust=True, + base="ubuntu@22.04", ), ops_test.model.deploy( - "kafka-k8s", application_name="kafka", trust=True, channel="latest/stable" + "kafka-k8s", + application_name="kafka", + trust=True, + channel="latest/stable", + base="ubuntu@20.04", ), ops_test.model.deploy( - "zookeeper-k8s", application_name="zookeeper", channel="latest/stable" + "zookeeper-k8s", + application_name="zookeeper", + channel="latest/stable", + base="ubuntu@20.04", ), # sticking to revision that support both juju 2.9.x and 3.x ops_test.model.deploy( diff --git a/tests/integration/test_backup_aws.py b/tests/integration/test_backup_aws.py index c5da55db6..fc63b1eca 100644 --- a/tests/integration/test_backup_aws.py +++ b/tests/integration/test_backup_aws.py @@ -98,7 +98,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: logger.info("Deploying s3-integrator") - await ops_test.model.deploy(S3_INTEGRATOR, channel="stable") + await ops_test.model.deploy(S3_INTEGRATOR, channel="stable", base="ubuntu@22.04") await ops_test.model.relate(mysql_application_name, S3_INTEGRATOR) await ops_test.model.wait_for_idle( diff --git a/tests/integration/test_backup_ceph.py b/tests/integration/test_backup_ceph.py index 15025262d..161e2500f 100644 --- a/tests/integration/test_backup_ceph.py +++ b/tests/integration/test_backup_ceph.py @@ -94,7 +94,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: logger.info("Deploying s3-integrator") - await ops_test.model.deploy(S3_INTEGRATOR, channel="stable") + await ops_test.model.deploy(S3_INTEGRATOR, channel="stable", base="ubuntu@22.04") await ops_test.model.relate(mysql_application_name, S3_INTEGRATOR) await ops_test.model.wait_for_idle( diff --git a/tests/integration/test_backup_gcp.py b/tests/integration/test_backup_gcp.py index c4fb55bb1..bb927914b 100644 --- a/tests/integration/test_backup_gcp.py +++ b/tests/integration/test_backup_gcp.py @@ -98,7 +98,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: logger.info("Deploying s3-integrator") - await ops_test.model.deploy(S3_INTEGRATOR, channel="stable") + await ops_test.model.deploy(S3_INTEGRATOR, channel="stable", base="ubuntu@22.04") await ops_test.model.relate(mysql_application_name, S3_INTEGRATOR) await ops_test.model.wait_for_idle( diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index e5a8a80a4..a3ba4274f 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -58,7 +58,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: application_name=APP_NAME, config=config, num_units=3, - series="jammy", + base="ubuntu@22.04", trust=True, ) diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index d8f68e5ad..f445fb98a 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -69,7 +69,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: resources=resources, application_name=APP_NAME, num_units=3, - series="jammy", + base="ubuntu@22.04", trust=True, config=config, ) @@ -128,7 +128,9 @@ async def test_enable_tls(ops_test: OpsTest) -> None: # Deploy TLS Certificates operator. logger.info("Deploy TLS operator") async with ops_test.fast_forward("60s"): - await ops_test.model.deploy(tls_app_name, channel=tls_channel, config=tls_config) + await ops_test.model.deploy( + tls_app_name, channel=tls_channel, config=tls_config, base="ubuntu@22.04" + ) await ops_test.model.wait_for_idle(apps=[tls_app_name], status="active", timeout=15 * 60) # Relate with TLS charm