diff --git a/.github/workflows/build_charm.md b/.github/workflows/build_charm.md index 33e0bcae..5f6db7cd 100644 --- a/.github/workflows/build_charm.md +++ b/.github/workflows/build_charm.md @@ -16,3 +16,8 @@ with: cache: true ``` remember to add your charm's branch(es) to charmcraftcache by running `ccc add` or by [opening an issue](https://github.com/canonical/charmcraftcache-hub/issues/new?assignees=&labels=add-charm&projects=&template=add_charm_branch.yaml&title=Add+charm+branch). + +### Required charmcraft.yaml syntax +Only [ST124 - Multi-base platforms in craft tools](https://docs.google.com/document/d/1QVHxZumruKVZ3yJ2C74qWhvs-ye5I9S6avMBDHs2YcQ/edit) "shorthand notation" syntax is supported in charmcraft.yaml + +Follow [step #1 from charmcraftst124's documentation](https://github.com/canonical/charmcraftst124?tab=readme-ov-file#step-1-update-charmcraftyaml-to-supported-syntax) diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 6cac8df8..35504031 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -51,11 +51,11 @@ on: outputs: artifact-prefix: description: Charm packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: - name: Collect bases for charm | ${{ inputs.path-to-charm-directory }} + collect-platforms: + name: Collect platforms for charm | ${{ inputs.path-to-charm-directory }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -68,27 +68,32 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install CLI run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli + - name: Install charmcraftst124 + run: pipx install charmcraftst124 - name: Checkout uses: actions/checkout@v4 - - name: Collect charm bases to build from charmcraft.yaml + - name: Check if supported ST124 shorthand notation syntax is used in charmcraft.yaml + working-directory: ${{ inputs.path-to-charm-directory }} + run: charmcraftst124 check-charmcraft-yaml -v + - name: Collect charm platforms to build from charmcraft.yaml id: collect - run: collect-charm-bases --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' + run: collect-charm-platforms --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build charm | base #${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build charm | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 120 steps: - name: (GitHub-hosted ARM runner) Install libpq-dev - if: ${{ matrix.base.runner == 'Ubuntu_ARM64_4C_16G_02' }} + if: ${{ matrix.platform.runner == 'Ubuntu_ARM64_4C_16G_02' }} # Needed for `charmcraftcache` to resolve dependencies (for postgresql charms with psycopg2) run: | sudo apt-get update @@ -136,6 +141,7 @@ jobs: poetry config warnings.export false pipx install charmcraftcache + pipx install charmcraftst124 - run: snap list - name: Pack charm id: pack @@ -143,12 +149,10 @@ jobs: run: | if '${{ inputs.cache }}' then - sg lxd -c "charmcraftcache pack -v --bases-index='${{ matrix.base.id }}'" + echo 'Cache not yet supported with ST124 syntax' + exit 1 else - # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2 - touch requirements.txt - - sg lxd -c "charmcraft pack -v --bases-index='${{ matrix.base.id }}'" + sg lxd -c "charmcraftst124 pack -v --platform='${{ matrix.platform.name }}'" fi env: # Used by charmcraftcache (to avoid GitHub API rate limit) @@ -157,14 +161,14 @@ jobs: if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-charmcraft-build-${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }} + name: logs-charmcraft-build-${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }} path: ~/.local/state/charmcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload charm package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py deleted file mode 100644 index 22c6d85a..00000000 --- a/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -"""Collect bases to build - -charmcraft: "bases" -snapcraft: "architectures" -rockcraft: "platforms" - -snaps & rocks are usually built on multiple architectures but only one Ubuntu version/base -charms (subordinate) can be built on multiple Ubuntu versions -""" - -import argparse -import json -import logging -import pathlib -import sys - -import yaml - -from .. import github_actions -from . import craft - -logging.basicConfig(level=logging.INFO, stream=sys.stdout) -RUNNERS = { - craft.Architecture.X64: "ubuntu-latest", - craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02", -} - - -def get_bases(*, craft_: craft.Craft, yaml_data): - """Get architecture for each base - - For charms, multiple bases can have the same architecture - (e.g. Ubuntu 20.04 X64 and Ubuntu 22.04 X64) - - For snaps & rocks, the Ubuntu version is the same for all architectures. - """ - if craft_ is craft.Craft.ROCK: - # https://canonical-rockcraft.readthedocs-hosted.com/en/latest/reference/rockcraft.yaml/#platforms - return [craft.Architecture(arch) for arch in yaml_data["platforms"]] - if craft_ is craft.Craft.SNAP: - bases = yaml_data.get("architectures") - if not bases: - # Default to X64 - return [craft.Architecture.X64] - elif craft_ is craft.Craft.CHARM: - bases = yaml_data["bases"] - else: - raise ValueError - arch_for_bases = [] - for platform in bases: - if craft_ is craft.Craft.SNAP: - # https://snapcraft.io/docs/explanation-architectures - build_on_architectures = platform["build-on"] - elif craft_ is craft.Craft.CHARM: - # https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 - build_on = platform.get("build-on") - if build_on: - assert isinstance(build_on, list) and len(build_on) == 1 - platform = build_on[0] - build_on_architectures = platform.get("architectures") - if not build_on_architectures: - # Default to X64 - arch_for_bases.append(craft.Architecture.X64) - continue - else: - raise ValueError - assert ( - len(build_on_architectures) == 1 - ), f"Multiple architectures ({build_on_architectures}) in one ({craft_.value}craft.yaml) base/architecture entry not supported. Use one entry per architecture" - arch_for_bases.append(craft.Architecture(build_on_architectures[0])) - return arch_for_bases - - -def collect(craft_: craft.Craft): - """Collect bases to build from *craft.yaml""" - parser = argparse.ArgumentParser() - parser.add_argument("--directory", required=True) - if craft_ is craft.Craft.CHARM: - parser.add_argument("--cache", required=True) - args = parser.parse_args() - craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml") - if craft_ is craft.Craft.SNAP: - craft_file = craft_file.parent / "snap" / craft_file.name - yaml_data = yaml.safe_load(craft_file.read_text()) - bases_ = get_bases(craft_=craft_, yaml_data=yaml_data) - bases = [] - for index, architecture in enumerate(bases_): - # id used to select base in `*craft pack` - if craft_ is craft.Craft.CHARM: - id_ = index - else: - id_ = architecture.value - bases.append({"id": id_, "runner": RUNNERS[architecture]}) - github_actions.output["bases"] = json.dumps(bases) - default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' - if craft_ is craft.Craft.CHARM: - default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' - github_actions.output["default_prefix"] = default_prefix - - -def snap(): - collect(craft.Craft.SNAP) - - -def rock(): - collect(craft.Craft.ROCK) - - -def charm(): - collect(craft.Craft.CHARM) diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py new file mode 100644 index 00000000..7bbff23d --- /dev/null +++ b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py @@ -0,0 +1,68 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +"""Collect ST124 shorthand notation platforms to build from charmcraft.yaml + +TODO add ST124 support for snaps & rocks +""" + +import argparse +import json +import logging +import pathlib +import sys + +import yaml + +from .. import github_actions +from . import craft + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) +RUNNERS = { + craft.Architecture.X64: "ubuntu-latest", + craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02", +} + + +def collect(craft_: craft.Craft): + """Collect platforms to build from *craft.yaml""" + parser = argparse.ArgumentParser() + parser.add_argument("--directory", required=True) + if craft_ is craft.Craft.CHARM: + parser.add_argument("--cache", required=True) + args = parser.parse_args() + craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml") + if craft_ is craft.Craft.SNAP: + craft_file = craft_file.parent / "snap" / craft_file.name + yaml_data = yaml.safe_load(craft_file.read_text()) + if craft_ is craft.Craft.CHARM: + # todo: run ccst124 validate + platforms = [] + for platform in yaml_data["platforms"]: + # Example `platform`: "ubuntu@22.04:amd64" + architecture = craft.Architecture(platform.split(":")[-1]) + platforms.append( + { + "name": platform, + "runner": RUNNERS[architecture], + "name_in_artifact": platform.replace(":", "-"), + } + ) + github_actions.output["platforms"] = json.dumps(platforms) + else: + raise ValueError("ST124 syntax not yet supported for snaps or rocks") + default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' + if craft_ is craft.Craft.CHARM: + default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' + github_actions.output["default_prefix"] = default_prefix + + +def snap(): + collect(craft.Craft.SNAP) + + +def rock(): + collect(craft.Craft.ROCK) + + +def charm(): + collect(craft.Craft.CHARM) diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index eb4b4f9f..9029c18a 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -9,9 +9,9 @@ readme = "README.md" [tool.poetry.scripts] redact-secrets = "data_platform_workflows_cli.redact_secrets:main" -collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_bases:snap" -collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_bases:rock" -collect-charm-bases = "data_platform_workflows_cli.craft_tools.collect_bases:charm" +collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" +collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" +collect-charm-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:charm" release-snap = "data_platform_workflows_cli.craft_tools.release:snap" release-rock = "data_platform_workflows_cli.craft_tools.release:rock" release-charm = "data_platform_workflows_cli.craft_tools.release:charm" diff --git a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py index f7e1b26d..3bbf12b7 100644 --- a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py +++ b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py @@ -3,8 +3,6 @@ import subprocess import typing -import yaml - def pytest_configure(config): if os.environ.get("CI") == "true": @@ -19,9 +17,7 @@ def pytest_configure(config): ) -async def build_charm( - self, charm_path: typing.Union[str, os.PathLike], bases_index: int = None -) -> pathlib.Path: +async def build_charm(self, charm_path: typing.Union[str, os.PathLike]) -> pathlib.Path: charm_path = pathlib.Path(charm_path) architecture = subprocess.run( ["dpkg", "--print-architecture"], @@ -30,24 +26,8 @@ async def build_charm( encoding="utf-8", ).stdout.strip() assert architecture in ("amd64", "arm64") - if bases_index is not None: - charmcraft_yaml = yaml.safe_load((charm_path / "charmcraft.yaml").read_text()) - assert charmcraft_yaml["type"] == "charm" - base = charmcraft_yaml["bases"][bases_index] - # Handle multiple base formats - # See https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 - build_on = base.get("build-on", [base])[0] - version = build_on["channel"] - architectures = build_on.get("architectures", ["amd64"]) - assert ( - len(architectures) == 1 - ), f"Multiple architectures ({architectures}) in one (charmcraft.yaml) base not supported. Use one base per architecture" - assert ( - architectures[0] == architecture - ), f"Architecture for {bases_index=} ({architectures[0]}) does not match host architecture ({architecture})" - packed_charms = list(charm_path.glob(f"*{version}-{architecture}.charm")) - else: - packed_charms = list(charm_path.glob(f"*-{architecture}.charm")) + # TODO unpin 22.04 (temporary solution while multi-base integration testing not supported by data-platform-workflows) + packed_charms = list(charm_path.glob(f"*-22.04-{architecture}.charm")) if len(packed_charms) == 1: # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms # to begin with `./` or `/` to distinguish them from Charmhub charms. @@ -58,11 +38,11 @@ async def build_charm( # `pathlib.Path`.) return packed_charms[0].resolve(strict=True) elif len(packed_charms) > 1: - message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}." - if bases_index is None: - message += " Specify `bases_index`" - raise ValueError(message) + raise ValueError( + f"More than one matching .charm file found at {charm_path=} for {architecture=} and " + f"Ubuntu 22.04: {packed_charms}." + ) else: raise ValueError( - f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}" + f"Unable to find .charm file for {architecture=} and Ubuntu 22.04 at {charm_path=}" )