Support ST124 shorthand notation syntax in charmcraft.yaml

Enables building & releasing multi-base charms with 24.04 in a single charmcraft.yaml and git branch

Integration testing is not supported on multiple bases—it is currently only supported on 22.04

.github/workflows/build_charm.md

@@ -16,3 +16,8 @@ with:
   cache: true
 ```
 remember to add your charm's branch(es) to charmcraftcache by running `ccc add` or by [opening an issue](https://github.com/canonical/charmcraftcache-hub/issues/new?assignees=&labels=add-charm&projects=&template=add_charm_branch.yaml&title=Add+charm+branch).
+
+### Required charmcraft.yaml syntax
+Only [ST124 - Multi-base platforms in craft tools](https://docs.google.com/document/d/1QVHxZumruKVZ3yJ2C74qWhvs-ye5I9S6avMBDHs2YcQ/edit) "shorthand notation" syntax is supported in charmcraft.yaml
+
+Follow [step #1 from charmcraftst124's documentation](https://github.com/canonical/charmcraftst124?tab=readme-ov-file#step-1-update-charmcraftyaml-to-supported-syntax)

.github/workflows/build_charm.yaml

@@ -51,11 +51,11 @@ on:
     outputs:
       artifact-prefix:
         description: Charm packages are uploaded to GitHub artifacts beginning with this prefix
-        value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }}
+        value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }}
 
 jobs:
-  collect-bases:
-    name: Collect bases for charm | ${{ inputs.path-to-charm-directory }}
+  collect-platforms:
+    name: Collect platforms for charm | ${{ inputs.path-to-charm-directory }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -68,27 +68,32 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install CLI
         run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli
+      - name: Install charmcraftst124
+        run: pipx install charmcraftst124
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Collect charm bases to build from charmcraft.yaml
+      - name: Check if supported ST124 shorthand notation syntax is used in charmcraft.yaml
+        working-directory: ${{ inputs.path-to-charm-directory }}
+        run: charmcraftst124 check-charmcraft-yaml -v
+      - name: Collect charm platforms to build from charmcraft.yaml
         id: collect
-        run: collect-charm-bases --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}'
+        run: collect-charm-platforms --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}'
     outputs:
-      bases: ${{ steps.collect.outputs.bases }}
+      platforms: ${{ steps.collect.outputs.platforms }}
       artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }}
 
   build:
     strategy:
       matrix:
-        base: ${{ fromJSON(needs.collect-bases.outputs.bases) }}
-    name: 'Build charm | base #${{ matrix.base.id }}'
+        platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }}
+    name: 'Build charm | ${{ matrix.platform.name }}'
     needs:
-      - collect-bases
-    runs-on: ${{ matrix.base.runner }}
+      - collect-platforms
+    runs-on: ${{ matrix.platform.runner }}
     timeout-minutes: 120
     steps:
       - name: (GitHub-hosted ARM runner) Install libpq-dev
-        if: ${{ matrix.base.runner == 'Ubuntu_ARM64_4C_16G_02' }}
+        if: ${{ matrix.platform.runner == 'Ubuntu_ARM64_4C_16G_02' }}
         # Needed for `charmcraftcache` to resolve dependencies (for postgresql charms with psycopg2)
         run: |
           sudo apt-get update
@@ -136,19 +141,18 @@ jobs:
           poetry config warnings.export false
 
           pipx install charmcraftcache
+          pipx install charmcraftst124
       - run: snap list
       - name: Pack charm
         id: pack
         working-directory: ${{ inputs.path-to-charm-directory }}
         run: |
           if '${{ inputs.cache }}'
           then
-            sg lxd -c "charmcraftcache pack -v --bases-index='${{ matrix.base.id }}'"
+            echo 'Cache not yet supported with ST124 syntax'
+            exit 1
           else
-            # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2
-            touch requirements.txt
-
-            sg lxd -c "charmcraft pack -v --bases-index='${{ matrix.base.id }}'"
+            sg lxd -c "charmcraftst124 pack -v --platform='${{ matrix.platform.name }}'"
           fi
         env:
           # Used by charmcraftcache (to avoid GitHub API rate limit)
@@ -157,14 +161,14 @@ jobs:
         if: ${{ failure() && steps.pack.outcome == 'failure' }}
         uses: actions/upload-artifact@v4
         with:
-          name: logs-charmcraft-build-${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }}
+          name: logs-charmcraft-build-${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }}
           path: ~/.local/state/charmcraft/log/
           if-no-files-found: error
       - run: touch .empty
       - name: Upload charm package
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }}
+          name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }}
           # .empty file required to preserve directory structure
           # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156
           path: |

python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py

@@ -0,0 +1,68 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Collect ST124 shorthand notation platforms to build from charmcraft.yaml
+
+TODO add ST124 support for snaps & rocks
+"""
+
+import argparse
+import json
+import logging
+import pathlib
+import sys
+
+import yaml
+
+from .. import github_actions
+from . import craft
+
+logging.basicConfig(level=logging.INFO, stream=sys.stdout)
+RUNNERS = {
+    craft.Architecture.X64: "ubuntu-latest",
+    craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02",
+}
+
+
+def collect(craft_: craft.Craft):
+    """Collect platforms to build from *craft.yaml"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--directory", required=True)
+    if craft_ is craft.Craft.CHARM:
+        parser.add_argument("--cache", required=True)
+    args = parser.parse_args()
+    craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml")
+    if craft_ is craft.Craft.SNAP:
+        craft_file = craft_file.parent / "snap" / craft_file.name
+    yaml_data = yaml.safe_load(craft_file.read_text())
+    if craft_ is craft.Craft.CHARM:
+        # todo: run ccst124 validate
+        platforms = []
+        for platform in yaml_data["platforms"]:
+            # Example `platform`: "[email protected]:amd64"
+            architecture = craft.Architecture(platform.split(":")[-1])
+            platforms.append(
+                {
+                    "name": platform,
+                    "runner": RUNNERS[architecture],
+                    "name_in_artifact": platform.replace(":", "-"),
+                }
+            )
+        github_actions.output["platforms"] = json.dumps(platforms)
+    else:
+        raise ValueError("ST124 syntax not yet supported for snaps or rocks")
+    default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}'
+    if craft_ is craft.Craft.CHARM:
+        default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}'
+    github_actions.output["default_prefix"] = default_prefix
+
+
+def snap():
+    collect(craft.Craft.SNAP)
+
+
+def rock():
+    collect(craft.Craft.ROCK)
+
+
+def charm():
+    collect(craft.Craft.CHARM)

python/cli/pyproject.toml

@@ -9,9 +9,9 @@ readme = "README.md"
 
 [tool.poetry.scripts]
 redact-secrets = "data_platform_workflows_cli.redact_secrets:main"
-collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_bases:snap"
-collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_bases:rock"
-collect-charm-bases = "data_platform_workflows_cli.craft_tools.collect_bases:charm"
+collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:snap"
+collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:rock"
+collect-charm-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:charm"
 release-snap = "data_platform_workflows_cli.craft_tools.release:snap"
 release-rock = "data_platform_workflows_cli.craft_tools.release:rock"
 release-charm = "data_platform_workflows_cli.craft_tools.release:charm"

python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py

@@ -3,8 +3,6 @@
 import subprocess
 import typing
 
-import yaml
-
 
 def pytest_configure(config):
     if os.environ.get("CI") == "true":
@@ -19,9 +17,7 @@ def pytest_configure(config):
         )
 
 
-async def build_charm(
-    self, charm_path: typing.Union[str, os.PathLike], bases_index: int = None
-) -> pathlib.Path:
+async def build_charm(self, charm_path: typing.Union[str, os.PathLike]) -> pathlib.Path:
     charm_path = pathlib.Path(charm_path)
     architecture = subprocess.run(
         ["dpkg", "--print-architecture"],
@@ -30,24 +26,8 @@ async def build_charm(
         encoding="utf-8",
     ).stdout.strip()
     assert architecture in ("amd64", "arm64")
-    if bases_index is not None:
-        charmcraft_yaml = yaml.safe_load((charm_path / "charmcraft.yaml").read_text())
-        assert charmcraft_yaml["type"] == "charm"
-        base = charmcraft_yaml["bases"][bases_index]
-        # Handle multiple base formats
-        # See https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713
-        build_on = base.get("build-on", [base])[0]
-        version = build_on["channel"]
-        architectures = build_on.get("architectures", ["amd64"])
-        assert (
-            len(architectures) == 1
-        ), f"Multiple architectures ({architectures}) in one (charmcraft.yaml) base not supported. Use one base per architecture"
-        assert (
-            architectures[0] == architecture
-        ), f"Architecture for {bases_index=} ({architectures[0]}) does not match host architecture ({architecture})"
-        packed_charms = list(charm_path.glob(f"*{version}-{architecture}.charm"))
-    else:
-        packed_charms = list(charm_path.glob(f"*-{architecture}.charm"))
+    # TODO unpin 22.04 (temporary solution while multi-base integration testing not supported by data-platform-workflows)
+    packed_charms = list(charm_path.glob(f"*-22.04-{architecture}.charm"))
     if len(packed_charms) == 1:
         # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms
         # to begin with `./` or `/` to distinguish them from Charmhub charms.
@@ -58,11 +38,11 @@ async def build_charm(
         # `pathlib.Path`.)
         return packed_charms[0].resolve(strict=True)
     elif len(packed_charms) > 1:
-        message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}."
-        if bases_index is None:
-            message += " Specify `bases_index`"
-        raise ValueError(message)
+        raise ValueError(
+            f"More than one matching .charm file found at {charm_path=} for {architecture=} and "
+            f"Ubuntu 22.04: {packed_charms}."
+        )
     else:
         raise ValueError(
-            f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}"
+            f"Unable to find .charm file for {architecture=} and Ubuntu 22.04 at {charm_path=}"
         )