From 62babbfdef20dc1ad8e4ae8ddf62a70886b896d7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 5 Dec 2024 11:29:03 +0100 Subject: [PATCH 1/7] Support ST124 shorthand notation syntax in charmcraft.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enables building & releasing multi-base charms with 24.04 in a single charmcraft.yaml and git branch Integration testing is not supported on multiple bases—it is currently only supported on 22.04 --- .github/workflows/build_charm.md | 20 ++++ .github/workflows/build_charm.yaml | 40 ++++--- .../craft_tools/collect_bases.py | 112 ------------------ .../craft_tools/collect_platforms.py | 68 +++++++++++ python/cli/pyproject.toml | 6 +- .../pytest_operator_cache/_plugin.py | 36 ++---- 6 files changed, 121 insertions(+), 161 deletions(-) delete mode 100644 python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py create mode 100644 python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py diff --git a/.github/workflows/build_charm.md b/.github/workflows/build_charm.md index 33e0bcae..7e64fb61 100644 --- a/.github/workflows/build_charm.md +++ b/.github/workflows/build_charm.md @@ -16,3 +16,23 @@ with: cache: true ``` remember to add your charm's branch(es) to charmcraftcache by running `ccc add` or by [opening an issue](https://github.com/canonical/charmcraftcache-hub/issues/new?assignees=&labels=add-charm&projects=&template=add_charm_branch.yaml&title=Add+charm+branch). + +### Required charmcraft.yaml syntax +Only [ST124 - Multi-base platforms in craft tools](https://docs.google.com/document/d/1QVHxZumruKVZ3yJ2C74qWhvs-ye5I9S6avMBDHs2YcQ/edit) "shorthand notation" syntax is supported + +#### Example +```yaml +platforms: + ubuntu@22.04:amd64: + ubuntu@22.04:arm64: + ubuntu@24.04:amd64: + ubuntu@24.04:arm64: +``` + +Under the charmcraft.yaml `platforms` key, `build-on` and `build-for` syntax are not supported + +The `base` and `bases` charmcraft.yaml keys are not supported + +> [!NOTE] +> ST124 will be implemented in a future version of charmcraft. Until then, use charmcraftcache or [charmcraftst124](https://github.com/canonical/charmcraftst124) if you need to pack an ST124 charm outside of CI. + diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 6cac8df8..35504031 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -51,11 +51,11 @@ on: outputs: artifact-prefix: description: Charm packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: - name: Collect bases for charm | ${{ inputs.path-to-charm-directory }} + collect-platforms: + name: Collect platforms for charm | ${{ inputs.path-to-charm-directory }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -68,27 +68,32 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install CLI run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli + - name: Install charmcraftst124 + run: pipx install charmcraftst124 - name: Checkout uses: actions/checkout@v4 - - name: Collect charm bases to build from charmcraft.yaml + - name: Check if supported ST124 shorthand notation syntax is used in charmcraft.yaml + working-directory: ${{ inputs.path-to-charm-directory }} + run: charmcraftst124 check-charmcraft-yaml -v + - name: Collect charm platforms to build from charmcraft.yaml id: collect - run: collect-charm-bases --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' + run: collect-charm-platforms --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build charm | base #${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build charm | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 120 steps: - name: (GitHub-hosted ARM runner) Install libpq-dev - if: ${{ matrix.base.runner == 'Ubuntu_ARM64_4C_16G_02' }} + if: ${{ matrix.platform.runner == 'Ubuntu_ARM64_4C_16G_02' }} # Needed for `charmcraftcache` to resolve dependencies (for postgresql charms with psycopg2) run: | sudo apt-get update @@ -136,6 +141,7 @@ jobs: poetry config warnings.export false pipx install charmcraftcache + pipx install charmcraftst124 - run: snap list - name: Pack charm id: pack @@ -143,12 +149,10 @@ jobs: run: | if '${{ inputs.cache }}' then - sg lxd -c "charmcraftcache pack -v --bases-index='${{ matrix.base.id }}'" + echo 'Cache not yet supported with ST124 syntax' + exit 1 else - # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2 - touch requirements.txt - - sg lxd -c "charmcraft pack -v --bases-index='${{ matrix.base.id }}'" + sg lxd -c "charmcraftst124 pack -v --platform='${{ matrix.platform.name }}'" fi env: # Used by charmcraftcache (to avoid GitHub API rate limit) @@ -157,14 +161,14 @@ jobs: if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-charmcraft-build-${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }} + name: logs-charmcraft-build-${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }} path: ~/.local/state/charmcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload charm package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py deleted file mode 100644 index 22c6d85a..00000000 --- a/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -"""Collect bases to build - -charmcraft: "bases" -snapcraft: "architectures" -rockcraft: "platforms" - -snaps & rocks are usually built on multiple architectures but only one Ubuntu version/base -charms (subordinate) can be built on multiple Ubuntu versions -""" - -import argparse -import json -import logging -import pathlib -import sys - -import yaml - -from .. import github_actions -from . import craft - -logging.basicConfig(level=logging.INFO, stream=sys.stdout) -RUNNERS = { - craft.Architecture.X64: "ubuntu-latest", - craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02", -} - - -def get_bases(*, craft_: craft.Craft, yaml_data): - """Get architecture for each base - - For charms, multiple bases can have the same architecture - (e.g. Ubuntu 20.04 X64 and Ubuntu 22.04 X64) - - For snaps & rocks, the Ubuntu version is the same for all architectures. - """ - if craft_ is craft.Craft.ROCK: - # https://canonical-rockcraft.readthedocs-hosted.com/en/latest/reference/rockcraft.yaml/#platforms - return [craft.Architecture(arch) for arch in yaml_data["platforms"]] - if craft_ is craft.Craft.SNAP: - bases = yaml_data.get("architectures") - if not bases: - # Default to X64 - return [craft.Architecture.X64] - elif craft_ is craft.Craft.CHARM: - bases = yaml_data["bases"] - else: - raise ValueError - arch_for_bases = [] - for platform in bases: - if craft_ is craft.Craft.SNAP: - # https://snapcraft.io/docs/explanation-architectures - build_on_architectures = platform["build-on"] - elif craft_ is craft.Craft.CHARM: - # https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 - build_on = platform.get("build-on") - if build_on: - assert isinstance(build_on, list) and len(build_on) == 1 - platform = build_on[0] - build_on_architectures = platform.get("architectures") - if not build_on_architectures: - # Default to X64 - arch_for_bases.append(craft.Architecture.X64) - continue - else: - raise ValueError - assert ( - len(build_on_architectures) == 1 - ), f"Multiple architectures ({build_on_architectures}) in one ({craft_.value}craft.yaml) base/architecture entry not supported. Use one entry per architecture" - arch_for_bases.append(craft.Architecture(build_on_architectures[0])) - return arch_for_bases - - -def collect(craft_: craft.Craft): - """Collect bases to build from *craft.yaml""" - parser = argparse.ArgumentParser() - parser.add_argument("--directory", required=True) - if craft_ is craft.Craft.CHARM: - parser.add_argument("--cache", required=True) - args = parser.parse_args() - craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml") - if craft_ is craft.Craft.SNAP: - craft_file = craft_file.parent / "snap" / craft_file.name - yaml_data = yaml.safe_load(craft_file.read_text()) - bases_ = get_bases(craft_=craft_, yaml_data=yaml_data) - bases = [] - for index, architecture in enumerate(bases_): - # id used to select base in `*craft pack` - if craft_ is craft.Craft.CHARM: - id_ = index - else: - id_ = architecture.value - bases.append({"id": id_, "runner": RUNNERS[architecture]}) - github_actions.output["bases"] = json.dumps(bases) - default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' - if craft_ is craft.Craft.CHARM: - default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' - github_actions.output["default_prefix"] = default_prefix - - -def snap(): - collect(craft.Craft.SNAP) - - -def rock(): - collect(craft.Craft.ROCK) - - -def charm(): - collect(craft.Craft.CHARM) diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py new file mode 100644 index 00000000..7bbff23d --- /dev/null +++ b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py @@ -0,0 +1,68 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +"""Collect ST124 shorthand notation platforms to build from charmcraft.yaml + +TODO add ST124 support for snaps & rocks +""" + +import argparse +import json +import logging +import pathlib +import sys + +import yaml + +from .. import github_actions +from . import craft + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) +RUNNERS = { + craft.Architecture.X64: "ubuntu-latest", + craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02", +} + + +def collect(craft_: craft.Craft): + """Collect platforms to build from *craft.yaml""" + parser = argparse.ArgumentParser() + parser.add_argument("--directory", required=True) + if craft_ is craft.Craft.CHARM: + parser.add_argument("--cache", required=True) + args = parser.parse_args() + craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml") + if craft_ is craft.Craft.SNAP: + craft_file = craft_file.parent / "snap" / craft_file.name + yaml_data = yaml.safe_load(craft_file.read_text()) + if craft_ is craft.Craft.CHARM: + # todo: run ccst124 validate + platforms = [] + for platform in yaml_data["platforms"]: + # Example `platform`: "ubuntu@22.04:amd64" + architecture = craft.Architecture(platform.split(":")[-1]) + platforms.append( + { + "name": platform, + "runner": RUNNERS[architecture], + "name_in_artifact": platform.replace(":", "-"), + } + ) + github_actions.output["platforms"] = json.dumps(platforms) + else: + raise ValueError("ST124 syntax not yet supported for snaps or rocks") + default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' + if craft_ is craft.Craft.CHARM: + default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' + github_actions.output["default_prefix"] = default_prefix + + +def snap(): + collect(craft.Craft.SNAP) + + +def rock(): + collect(craft.Craft.ROCK) + + +def charm(): + collect(craft.Craft.CHARM) diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index eb4b4f9f..9029c18a 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -9,9 +9,9 @@ readme = "README.md" [tool.poetry.scripts] redact-secrets = "data_platform_workflows_cli.redact_secrets:main" -collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_bases:snap" -collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_bases:rock" -collect-charm-bases = "data_platform_workflows_cli.craft_tools.collect_bases:charm" +collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" +collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" +collect-charm-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:charm" release-snap = "data_platform_workflows_cli.craft_tools.release:snap" release-rock = "data_platform_workflows_cli.craft_tools.release:rock" release-charm = "data_platform_workflows_cli.craft_tools.release:charm" diff --git a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py index f7e1b26d..3bbf12b7 100644 --- a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py +++ b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py @@ -3,8 +3,6 @@ import subprocess import typing -import yaml - def pytest_configure(config): if os.environ.get("CI") == "true": @@ -19,9 +17,7 @@ def pytest_configure(config): ) -async def build_charm( - self, charm_path: typing.Union[str, os.PathLike], bases_index: int = None -) -> pathlib.Path: +async def build_charm(self, charm_path: typing.Union[str, os.PathLike]) -> pathlib.Path: charm_path = pathlib.Path(charm_path) architecture = subprocess.run( ["dpkg", "--print-architecture"], @@ -30,24 +26,8 @@ async def build_charm( encoding="utf-8", ).stdout.strip() assert architecture in ("amd64", "arm64") - if bases_index is not None: - charmcraft_yaml = yaml.safe_load((charm_path / "charmcraft.yaml").read_text()) - assert charmcraft_yaml["type"] == "charm" - base = charmcraft_yaml["bases"][bases_index] - # Handle multiple base formats - # See https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 - build_on = base.get("build-on", [base])[0] - version = build_on["channel"] - architectures = build_on.get("architectures", ["amd64"]) - assert ( - len(architectures) == 1 - ), f"Multiple architectures ({architectures}) in one (charmcraft.yaml) base not supported. Use one base per architecture" - assert ( - architectures[0] == architecture - ), f"Architecture for {bases_index=} ({architectures[0]}) does not match host architecture ({architecture})" - packed_charms = list(charm_path.glob(f"*{version}-{architecture}.charm")) - else: - packed_charms = list(charm_path.glob(f"*-{architecture}.charm")) + # TODO unpin 22.04 (temporary solution while multi-base integration testing not supported by data-platform-workflows) + packed_charms = list(charm_path.glob(f"*-22.04-{architecture}.charm")) if len(packed_charms) == 1: # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms # to begin with `./` or `/` to distinguish them from Charmhub charms. @@ -58,11 +38,11 @@ async def build_charm( # `pathlib.Path`.) return packed_charms[0].resolve(strict=True) elif len(packed_charms) > 1: - message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}." - if bases_index is None: - message += " Specify `bases_index`" - raise ValueError(message) + raise ValueError( + f"More than one matching .charm file found at {charm_path=} for {architecture=} and " + f"Ubuntu 22.04: {packed_charms}." + ) else: raise ValueError( - f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}" + f"Unable to find .charm file for {architecture=} and Ubuntu 22.04 at {charm_path=}" ) From 0c19d3fc6be80a67471a53579f420cc825eac7ad Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 5 Dec 2024 16:56:43 +0100 Subject: [PATCH 2/7] Add back support for snap & rocks (not ST124 support) Adds support for core24 `platforms` in snapcraft.yaml --- .github/workflows/build_rock.md | 14 +++++- .github/workflows/build_rock.yaml | 22 +++++----- .github/workflows/build_snap.md | 33 +++++++++++++- .github/workflows/build_snap.yaml | 26 +++++------ .../craft_tools/collect_platforms.py | 44 ++++++++++++++++--- python/cli/pyproject.toml | 4 +- 6 files changed, 109 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build_rock.md b/.github/workflows/build_rock.md index c933765b..89d2eaa4 100644 --- a/.github/workflows/build_rock.md +++ b/.github/workflows/build_rock.md @@ -8,4 +8,16 @@ jobs: build: name: Build rock uses: canonical/data-platform-workflows/.github/workflows/build_rock.yaml@v0.0.0 -``` \ No newline at end of file +``` + +### Supported `platforms` syntax in rockcraft.yaml +Only "shorthand notation" is supported + +Example rockcraft.yaml +```yaml +platforms: + amd64: + arm64: +``` + +`build-on` and `build-for` are not supported diff --git a/.github/workflows/build_rock.yaml b/.github/workflows/build_rock.yaml index a46fa4a3..2e295984 100644 --- a/.github/workflows/build_rock.yaml +++ b/.github/workflows/build_rock.yaml @@ -28,10 +28,10 @@ on: outputs: artifact-prefix: description: Rock packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: + collect-platforms: name: Collect platforms for rock | ${{ inputs.path-to-rock-directory }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -49,19 +49,19 @@ jobs: uses: actions/checkout@v4 - name: Collect rock platforms to build from rockcraft.yaml id: collect - run: collect-rock-bases --directory='${{ inputs.path-to-rock-directory }}' + run: collect-rock-platforms --directory='${{ inputs.path-to-rock-directory }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build rock | ${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build rock | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 15 steps: - name: Get workflow version @@ -93,19 +93,19 @@ jobs: - name: Pack rock id: pack working-directory: ${{ inputs.path-to-rock-directory }} - run: sg lxd -c "rockcraft pack -v --platform='${{ matrix.base.id }}'" + run: sg lxd -c "rockcraft pack -v --platform='${{ matrix.platform.name }}'" - name: Upload rockcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-rockcraft-build-${{ inputs.artifact-prefix }}-architecture-${{ matrix.base.id }} + name: logs-rockcraft-build-${{ inputs.artifact-prefix }}-platform-${{ matrix.platform.name }} path: ~/.local/state/rockcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload rock package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-architecture-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/.github/workflows/build_snap.md b/.github/workflows/build_snap.md index 1ac88e00..181cf6a8 100644 --- a/.github/workflows/build_snap.md +++ b/.github/workflows/build_snap.md @@ -8,4 +8,35 @@ jobs: build: name: Build snap uses: canonical/data-platform-workflows/.github/workflows/build_snap.yaml@v0.0.0 -``` \ No newline at end of file +``` + +### Supported `platforms` and `architectures` syntax in snapcraft.yaml +See https://snapcraft.io/docs/architectures#how-to-create-a-snap-for-a-specific-architecture + +#### core24 +Only `platforms` is supported. `architectures` is not supported + +Only "shorthand notation" is supported + +Example snapcraft.yaml +```yaml +platforms: + amd64: + arm64: +``` + +`build-on` and `build-for` are not supported + +#### core22 +Only `architectures` is supported. `platforms` is not supported + +`architectures` must be a list of dictionaries. Each dictionary in the list must contain a `build-on` key + +Example snapcraft.yaml +```yaml +architectures: + - build-on: [amd64] + build-for: [amd64] + - build-on: [arm64] + build-for: [arm64] +``` diff --git a/.github/workflows/build_snap.yaml b/.github/workflows/build_snap.yaml index d7e46faa..f8ecee00 100644 --- a/.github/workflows/build_snap.yaml +++ b/.github/workflows/build_snap.yaml @@ -31,11 +31,11 @@ on: outputs: artifact-prefix: description: Snap packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: - name: Collect architectures for snap | ${{ inputs.path-to-snap-project-directory }} + collect-platforms: + name: Collect platforms for snap | ${{ inputs.path-to-snap-project-directory }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -50,21 +50,21 @@ jobs: run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - - name: Collect snap architectures to build from snapcraft.yaml + - name: Collect snap platforms to build from snapcraft.yaml id: collect - run: collect-snap-bases --directory='${{ inputs.path-to-snap-project-directory }}' + run: collect-snap-platforms --directory='${{ inputs.path-to-snap-project-directory }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build snap | ${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build snap | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 30 steps: - name: Get workflow version @@ -96,19 +96,19 @@ jobs: - name: Pack snap id: pack working-directory: ${{ inputs.path-to-snap-project-directory }} - run: sg lxd -c "snapcraft pack -v --build-for='${{ matrix.base.id }}'" + run: sg lxd -c "snapcraft pack -v --build-for='${{ matrix.platform.name }}'" - name: Upload snapcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-snapcraft-build-${{ inputs.artifact-prefix }}-architecture-${{ matrix.base.id }} + name: logs-snapcraft-build-${{ inputs.artifact-prefix }}-platform-${{ matrix.platform.name }} path: ~/.local/state/snapcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload snap package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-architecture-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py index 7bbff23d..82567a0f 100644 --- a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py +++ b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py @@ -1,8 +1,13 @@ # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. -"""Collect ST124 shorthand notation platforms to build from charmcraft.yaml +"""Collect platforms to build -TODO add ST124 support for snaps & rocks +charmcraft: Only ST124 shorthand notation `platforms` are supported +snapcraft: (ST124 not supported) core22 `architectures` and core24 shorthand `platforms` supported +rockcraft: (ST124 not supported) shorthand `platforms` supported + +snaps & rocks are usually built on multiple architectures but only one Ubuntu version/base +charms (subordinate) can be built on multiple Ubuntu versions """ import argparse @@ -34,9 +39,8 @@ def collect(craft_: craft.Craft): if craft_ is craft.Craft.SNAP: craft_file = craft_file.parent / "snap" / craft_file.name yaml_data = yaml.safe_load(craft_file.read_text()) + platforms = [] if craft_ is craft.Craft.CHARM: - # todo: run ccst124 validate - platforms = [] for platform in yaml_data["platforms"]: # Example `platform`: "ubuntu@22.04:amd64" architecture = craft.Architecture(platform.split(":")[-1]) @@ -47,9 +51,37 @@ def collect(craft_: craft.Craft): "name_in_artifact": platform.replace(":", "-"), } ) - github_actions.output["platforms"] = json.dumps(platforms) + elif craft_ is craft.Craft.ROCK: + for platform in yaml_data["platforms"]: + # Example `platform`: "amd64" + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + elif craft_ is craft.Craft.SNAP: + if yaml_data["base"] == "core24": + platforms_ = yaml_data["platforms"] + if not isinstance(platforms_, dict): + raise TypeError("Expected type 'dict' for snapcraft.yaml 'platforms'") + for value in platforms_.values(): + if value is not None: + raise ValueError( + "Only shorthand notation supported in snapcraft.yaml 'platforms'. " + "'build-on' and 'build-for' not supported" + ) + for platform in platforms_: + # Example `platform`: "amd64" + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + elif yaml_data["base"] == "core22": + for entry in yaml_data["architectures"]: + # Example: "amd64" + platform = entry["build-on"] + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + else: + raise ValueError(f'Unsupported snapcraft.yaml base: {repr(yaml_data["base"])}') else: - raise ValueError("ST124 syntax not yet supported for snaps or rocks") + raise ValueError + github_actions.output["platforms"] = json.dumps(platforms) default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' if craft_ is craft.Craft.CHARM: default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index 9029c18a..bf7dae2b 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -9,8 +9,8 @@ readme = "README.md" [tool.poetry.scripts] redact-secrets = "data_platform_workflows_cli.redact_secrets:main" -collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" -collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" +collect-snap-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" +collect-rock-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" collect-charm-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:charm" release-snap = "data_platform_workflows_cli.craft_tools.release:snap" release-rock = "data_platform_workflows_cli.craft_tools.release:rock" From 68b600e1ffde8b16d1e3afb4bf8a1cd8b64bc454 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 16 Dec 2024 16:12:19 +0100 Subject: [PATCH 3/7] Deprecate pytest-operator-cache --- .github/workflows/integration_test_charm.md | 4 +++- .../pytest_plugins/pytest_operator_cache/README.md | 3 +++ .../pytest_operator_cache/deprecation_notice.md | 14 ++++++++++++++ .../pytest_operator_cache/_plugin.py | 10 +++++++++- 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 python/pytest_plugins/pytest_operator_cache/deprecation_notice.md diff --git a/.github/workflows/integration_test_charm.md b/.github/workflows/integration_test_charm.md index 45c5ad33..fb2b9fdf 100644 --- a/.github/workflows/integration_test_charm.md +++ b/.github/workflows/integration_test_charm.md @@ -1,5 +1,8 @@ Workflow file: [integration_test_charm.yaml](integration_test_charm.yaml) +> [!WARNING] +> The `pytest-operator-cache` plugin is **deprecated**. Follow the migration instructions here: [pytest_operator_cache/deprecation_notice.md](../../python/pytest_plugins/pytest_operator_cache/deprecation_notice.md) + ## Usage ### Step 1: Create your workflow ```yaml @@ -27,7 +30,6 @@ jobs: #### Step A Add ```toml -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v0.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v0.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} ``` to your integration test dependencies in `pyproject.toml`. diff --git a/python/pytest_plugins/pytest_operator_cache/README.md b/python/pytest_plugins/pytest_operator_cache/README.md index 4060fca7..2175a180 100644 --- a/python/pytest_plugins/pytest_operator_cache/README.md +++ b/python/pytest_plugins/pytest_operator_cache/README.md @@ -1,3 +1,6 @@ +> [!WARNING] +> `pytest-operator-cache` is **deprecated**. Migration instructions: [deprecation_notice.md](deprecation_notice.md) + [pytest-operator](https://github.com/charmed-kubernetes/pytest-operator) plugin that overrides `ops_test.build_charm()` to return cached *.charm file instead of building new *.charm file. Usage: [integration_test_charm.md](../../../.github/workflows/integration_test_charm.md) \ No newline at end of file diff --git a/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md b/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md new file mode 100644 index 00000000..56139868 --- /dev/null +++ b/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md @@ -0,0 +1,14 @@ +# Deprecation notice + +`pytest-operator-cache` is deprecated & will be removed in a future release + +## Current behavior +With `pytest-operator-cache` installed, `ops_test.build_charm` (from [pytest-operator](https://github.com/charmed-kubernetes/pytest-operator)): +- On a local machine, builds the charm with charmcraft +- On CI (if `os.environ.get("CI") == "true"`), returns the path of an existing *.charm file—if the *.charm file was built on Ubuntu 22.04 + - (On CI, the *.charm files are built in a separate job [separate runner] from the integration tests) + +## Migration instructions +In integration tests, instead of calling `ops_test.build_charm`, assume the *.charm file exists and fail the test if the *.charm file is missing. The charm should be built outside of the test + +When running tests locally, if you would like the charm to be re-built each time the tests are run, consider using [charmcraftcache](https://github.com/canonical/charmcraftcache) (e.g. `ccc pack`) before the `pytest` command (e.g. in tox.ini). If you have multiple charms, `ccc pack` needs to be called once for each charm diff --git a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py index 3bbf12b7..04233ebb 100644 --- a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py +++ b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py @@ -2,9 +2,17 @@ import pathlib import subprocess import typing +import warnings def pytest_configure(config): + # TODO: use permalink + warnings.warn( + # "\n::warning::" for https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message + "\n::warning::The `pytest-operator-cache` plugin is deprecated. Follow the migration instructions here: " + "https://github.com/canonical/data-platform-workflows/blob/main/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md", + DeprecationWarning, + ) if os.environ.get("CI") == "true": # Running in GitHub Actions; skip build step plugin = config.pluginmanager.get_plugin("pytest-operator") @@ -26,7 +34,7 @@ async def build_charm(self, charm_path: typing.Union[str, os.PathLike]) -> pathl encoding="utf-8", ).stdout.strip() assert architecture in ("amd64", "arm64") - # TODO unpin 22.04 (temporary solution while multi-base integration testing not supported by data-platform-workflows) + # 22.04 pin is temporary solution while multi-base integration testing not supported by data-platform-workflows packed_charms = list(charm_path.glob(f"*-22.04-{architecture}.charm")) if len(packed_charms) == 1: # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms From f1605c54b5cafe6670ff6b4b859f2cc59cce87f7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 17 Dec 2024 10:35:09 +0100 Subject: [PATCH 4/7] Support cached builds --- .github/workflows/build_charm.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 35504031..350b10b7 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -149,8 +149,7 @@ jobs: run: | if '${{ inputs.cache }}' then - echo 'Cache not yet supported with ST124 syntax' - exit 1 + sg lxd -c "charmcraftcache pack -v --platform='${{ matrix.platform.name }}'" else sg lxd -c "charmcraftst124 pack -v --platform='${{ matrix.platform.name }}'" fi From ffb2230b853045a37187966c0106a31c32839aa1 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 17 Dec 2024 10:54:00 +0100 Subject: [PATCH 5/7] Move validation --- .github/workflows/build_charm.yaml | 5 -- .../craft_tools/charmcraft_platforms.py | 54 +++++++++++++++++++ .../craft_tools/collect_platforms.py | 6 +-- 3 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 350b10b7..b7654752 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -68,13 +68,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install CLI run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - - name: Install charmcraftst124 - run: pipx install charmcraftst124 - name: Checkout uses: actions/checkout@v4 - - name: Check if supported ST124 shorthand notation syntax is used in charmcraft.yaml - working-directory: ${{ inputs.path-to-charm-directory }} - run: charmcraftst124 check-charmcraft-yaml -v - name: Collect charm platforms to build from charmcraft.yaml id: collect run: collect-charm-platforms --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' diff --git a/python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py new file mode 100644 index 00000000..4650bbc1 --- /dev/null +++ b/python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py @@ -0,0 +1,54 @@ +# Copied from https://github.com/canonical/charmcraftcache/blob/main/charmcraftcache/_platforms.py +import pathlib + +import yaml + +_SYNTAX_DOCS = "https://github.com/canonical/data-platform-workflows/blob/main/.github/workflows/build_charm.md#required-charmcraftyaml-syntax" + + +class Platform(str): + """Platform in charmcraft.yaml 'platforms' (e.g. 'ubuntu@22.04:amd64')""" + + def __new__(cls, value: str, /): + try: + _, architecture = value.split(":") + except ValueError: + raise ValueError( + "Invalid ST124 shorthand notation in charmcraft.yaml 'platforms': " + f"{repr(value)}\n\nMore info: {_SYNTAX_DOCS}" + ) + instance: Platform = super().__new__(cls, value) + instance.architecture = architecture + return instance + + +def get(charmcraft_yaml: pathlib.Path, /): + """Get platforms from charmcraft.yaml""" + charmcraft_yaml_data = yaml.safe_load(charmcraft_yaml.read_text()) + for key in ("base", "bases"): + if key in charmcraft_yaml_data: + raise ValueError( + f"'{key}' key in charmcraft.yaml not supported. Use 'platforms' key instead.\n\n" + f"More info: {_SYNTAX_DOCS}" + ) + yaml_platforms = charmcraft_yaml_data.get("platforms") + if not yaml_platforms: + raise ValueError( + f"'platforms' key in charmcraft.yaml required\n\nMore info: {_SYNTAX_DOCS}" + ) + if not isinstance(yaml_platforms, dict): + raise TypeError( + "Expected charmcraft.yaml 'platforms' with type 'dict', got " + f"{repr(type(yaml_platforms).__name__)}: {repr(yaml_platforms)}\n\n" + f"More info: {_SYNTAX_DOCS}" + ) + for value in yaml_platforms.values(): + if value is not None: + raise ValueError( + "Shorthand notation required ('build-on' and 'build-for' not supported) in " + f"charmcraft.yaml 'platforms'.\n\nMore info: {_SYNTAX_DOCS}" + ) + # Validate format of keys in `yaml_platforms` + platforms = [Platform(platform) for platform in yaml_platforms] + + return platforms diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py index 82567a0f..385cb72e 100644 --- a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py +++ b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py @@ -20,6 +20,7 @@ from .. import github_actions from . import craft +from . import charmcraft_platforms logging.basicConfig(level=logging.INFO, stream=sys.stdout) RUNNERS = { @@ -41,13 +42,12 @@ def collect(craft_: craft.Craft): yaml_data = yaml.safe_load(craft_file.read_text()) platforms = [] if craft_ is craft.Craft.CHARM: - for platform in yaml_data["platforms"]: + for platform in charmcraft_platforms.get(craft_file): # Example `platform`: "ubuntu@22.04:amd64" - architecture = craft.Architecture(platform.split(":")[-1]) platforms.append( { "name": platform, - "runner": RUNNERS[architecture], + "runner": RUNNERS[platform.architecture], "name_in_artifact": platform.replace(":", "-"), } ) From 094257f6e0074bc7f7e4abc6c4ac0ed1ed8da7ae Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 17 Dec 2024 11:20:03 +0100 Subject: [PATCH 6/7] Remove old workaround --- .../cli/data_platform_workflows_cli/craft_tools/release.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/craft_tools/release.py b/python/cli/data_platform_workflows_cli/craft_tools/release.py index 80f6edb5..2a780945 100644 --- a/python/cli/data_platform_workflows_cli/craft_tools/release.py +++ b/python/cli/data_platform_workflows_cli/craft_tools/release.py @@ -193,11 +193,6 @@ def rock(): def charm(): - # Remove `charmcraft.yaml` from working directory (directory that subprocess will run as) if it - # exists. - # Workaround for https://github.com/canonical/charmcraft/issues/1389 - pathlib.Path("charmcraft.yaml").unlink(missing_ok=True) - parser = argparse.ArgumentParser() parser.add_argument("--directory", required=True) parser.add_argument("--channel", required=True) From cd58963755114cbbbd4033757704870f4bb291f4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 20 Dec 2024 09:51:39 +0100 Subject: [PATCH 7/7] Remove `charmcraftst124` --- .github/workflows/build_charm.md | 4 ---- .github/workflows/build_charm.yaml | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build_charm.md b/.github/workflows/build_charm.md index 7e64fb61..a8efcd96 100644 --- a/.github/workflows/build_charm.md +++ b/.github/workflows/build_charm.md @@ -32,7 +32,3 @@ platforms: Under the charmcraft.yaml `platforms` key, `build-on` and `build-for` syntax are not supported The `base` and `bases` charmcraft.yaml keys are not supported - -> [!NOTE] -> ST124 will be implemented in a future version of charmcraft. Until then, use charmcraftcache or [charmcraftst124](https://github.com/canonical/charmcraftst124) if you need to pack an ST124 charm outside of CI. - diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index b7654752..cc50ab56 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -136,7 +136,6 @@ jobs: poetry config warnings.export false pipx install charmcraftcache - pipx install charmcraftst124 - run: snap list - name: Pack charm id: pack @@ -146,7 +145,7 @@ jobs: then sg lxd -c "charmcraftcache pack -v --platform='${{ matrix.platform.name }}'" else - sg lxd -c "charmcraftst124 pack -v --platform='${{ matrix.platform.name }}'" + sg lxd -c "charmcraft pack -v --platform='${{ matrix.platform.name }}'" fi env: # Used by charmcraftcache (to avoid GitHub API rate limit)