From a78e496f6c5c85c0b8da19aaf1c1073e1f12a9ba Mon Sep 17 00:00:00 2001 From: Lucas Gameiro Date: Tue, 8 Oct 2024 10:31:00 -0300 Subject: [PATCH] patch: Add _update_bundle.yaml (#239) This PR aims to condense both previous patches #237 and #235 in a single, experimental `_update_bundle.yaml` workflow that will pin both snap and rock resource versions, in accordance to what was previously discussed: - For rocks: Roughly follows John Meinel's proposed format, with `oci-*` lines commented in order to preserve compatibility with `juju deploy`. See the following PR commit: https://github.com/canonical/postgresql-k8s-bundle/commit/2683a2d91004546787e2be8fc48bc81dc01d3a77 - For snaps: follows existing [store proxy standard](https://documentation.ubuntu.com/snap-store-proxy/en/airgap-charmhub/#export-snap-resources). See the following PR commit: https://github.com/canonical/postgresql-bundle/commit/b223dc3c8d176403cea67c441f5dadc99813b4f8 See also test runs on: - mysql bundle VM: https://github.com/canonical/mysql-bundle/pull/77/commits/612d5316706cadc45073290c1a55034617ccadca - mysql bundle K8s: https://github.com/canonical/mysql-k8s-bundle/pull/81/commits/018fae7197d47e049989390afb04e3f4ed510f48 Important info: - `storage-admin` currently has 2 blocking issues: - Pinning of charm revisions is being ignored: https://bugs.launchpad.net/snapstore-client/+bug/2083876 - Charm resource (e.g. rocks) version pinning not supported: https://bugs.launchpad.net/snapstore-client/+bug/2083878 - This is `amd64` only. An eventual patch for `arm64` should be straightforward. - This is specific to SQL bundles and their associated charms. This is the case because snap pinning requires specific logic per-charm, see point below. - This fetches snap revisions by individual parsing of each charm's source code (not by deploying the bundle, like on previous patch) --------- Co-authored-by: Carl Csaposs --- .github/workflows/_update_bundle.md | 26 ++ .github/workflows/_update_bundle.yaml | 80 +++++ README.md | 1 + .../update_bundle.py | 286 ++++++++++++++++++ 4 files changed, 393 insertions(+) create mode 100644 .github/workflows/_update_bundle.md create mode 100644 .github/workflows/_update_bundle.yaml create mode 100644 python/cli/data_platform_workflows_cli/update_bundle.py diff --git a/.github/workflows/_update_bundle.md b/.github/workflows/_update_bundle.md new file mode 100644 index 00000000..d7582d74 --- /dev/null +++ b/.github/workflows/_update_bundle.md @@ -0,0 +1,26 @@ +Workflow file: [_update_bundle.yaml](_update_bundle.yaml) + +> [!WARNING] +> Subject to **breaking changes on patch release**. `_update_bundle.yaml` is experimental & not part of the public interface. + +## Usage +Add `.yaml` file to `.github/workflows/` +```yaml +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +name: Update bundle + +on: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + +jobs: + update-bundle: + name: Update bundle + uses: canonical/data-platform-workflows/.github/workflows/_update_bundle.yaml@v0.0.0 + with: + path-to-bundle-file: bundle.yaml + reviewers: canonical/data-platform-engineers,octocat + secrets: + token: ${{ secrets.CREATE_PR_APP_TOKEN }} +``` diff --git a/.github/workflows/_update_bundle.yaml b/.github/workflows/_update_bundle.yaml new file mode 100644 index 00000000..376c8449 --- /dev/null +++ b/.github/workflows/_update_bundle.yaml @@ -0,0 +1,80 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +# Usage documentation: _update_bundle.md + +on: + workflow_call: + inputs: + path-to-bundle-file: + description: Relative path to bundle file from repository directory + required: true + type: string + reviewers: + description: Comma separated list of GitHub usernames to request to review pull request (e.g. "canonical/data-platform-engineers,octocat") + required: false + type: string + secrets: + token: + description: | + GitHub App token or personal access token (not GITHUB_TOKEN) + + Permissions needed for App token: + - Access: Read & write for Repository permissions: Pull requests + - Access: Read & write for Repository permissions: Contents + - If GitHub team is requested for pull request review, + Access: Read-only for Organization permissions: Members + + Permissions needed for personal access token: write access to repository, read:org + Personal access tokens with fine grained access are not supported (by GraphQL API, which is used by GitHub CLI). + + The GITHUB_TOKEN can create a pull request or push a branch, but `on: pull_request` workflows will not be triggered. + + Source: https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs + required: true + +jobs: + update-bundle: + name: Update bundle + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Get workflow version + id: workflow-version + uses: canonical/get-workflow-version-action@v1 + with: + repository-name: canonical/data-platform-workflows + file-name: _update_bundle.yaml + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install CLI + run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli + - name: Checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.token }} + - name: Update bundle file + id: update-file + run: update-bundle '${{ inputs.path-to-bundle-file }}' + - name: Push `update-bundle` branch + if: ${{ fromJSON(steps.update-file.outputs.updates_available) }} + run: | + git checkout -b update-bundle + git add . + git config user.name "GitHub Actions" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Update bundle" + # Uses token set in checkout step + git push origin update-bundle -f + - name: Create pull request + if: ${{ fromJSON(steps.update-file.outputs.updates_available) }} + run: | + # Capture output in variable so that step fails if `gh pr list` exits with non-zero code + prs=$(gh pr list --head update-bundle --state open --json number) + if [[ $prs != "[]" ]] + then + echo Open pull request already exists + exit 0 + fi + gh pr create --head update-bundle --title "Update bundle" --body "Update charm revisions in bundle YAML file" --reviewer '${{ inputs.reviewers }}' + env: + GH_TOKEN: ${{ secrets.token }} diff --git a/README.md b/README.md index 5cce2005..3cf970d6 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ | [release_rock.yaml](.github/workflows/release_rock.md) | Release rock to GitHub Container Registry | | [release_charm.yaml](.github/workflows/release_charm.md) | Release charm to Charmhub | | [sync_docs.yaml](.github/workflows/sync_docs.md) | Sync Discourse documentation to GitHub | +| [_update_bundle.yaml](.github/workflows/_update_bundle.md) | **Experimental** Update charm revisions in bundle | ### Version Recommendation: pin the latest version (e.g. `v1.0.0`) and use [Renovate](https://docs.renovatebot.com/) to stay up-to-date. diff --git a/python/cli/data_platform_workflows_cli/update_bundle.py b/python/cli/data_platform_workflows_cli/update_bundle.py new file mode 100644 index 00000000..58622e90 --- /dev/null +++ b/python/cli/data_platform_workflows_cli/update_bundle.py @@ -0,0 +1,286 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +"""Update charm revisions in bundle YAML file""" +import argparse +import ast +import copy +import dataclasses +import json +import pathlib +import re +import subprocess + +import requests +import yaml + +from . import github_actions + + +@dataclasses.dataclass(frozen=True) +class Snap: + name: str + revision: int + push_channel: str + + +def get_ubuntu_version(series: str) -> str: + """Gets Ubuntu version (e.g. "22.04") from series (e.g. "jammy").""" + return subprocess.run( + ["ubuntu-distro-info", "--series", series, "--release"], + capture_output=True, + check=True, + encoding="utf-8", + ).stdout.split(" ")[0] + + +def fetch_var_from_py_file(text, variable, safe=True): + """Parses .py file and returns the value assigned to a given variable inside it.""" + # While `ast.literal_eval` is safer and prefered, some vars are defined in terms of + # expressions (e.g. f-strings), not literals. In such cases, `exec` is the only way. + if not safe: + namespace = {} + exec(text, namespace) + return namespace[variable] + + parsed = ast.parse(text) + for node in ast.walk(parsed): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == variable: + return ast.literal_eval(node.value) + return None + + +def fetch_charm_info_from_store(charm, charm_channel) -> tuple[list[dict], list[dict]]: + """Returns, for a given channel, the necessary charm info from store endpoint.""" + response = requests.get( + f"https://api.snapcraft.io/v2/charms/info/{charm}?fields=channel-map,default-release&channel={charm_channel}" + ) + response.raise_for_status() + content = response.json() + return content["channel-map"], content["default-release"].get("resources", []) + + +def fetch_latest_charm_revision(channel_map, series=None) -> int | None: + """Gets the latest charm revision number in channel.""" + revisions = [] + for channel in channel_map: + if ( + channel["channel"]["base"]["architecture"] == "amd64" + and ( + series is None + or get_ubuntu_version(series) == channel["channel"]["base"]["channel"] + ) + ): + revisions.append(channel["revision"]["revision"]) + if not revisions: + return None + # If the charm supports multiple Ubuntu bases (and series=None), it's + # possible that there is a different revision for each base. + # Select the latest revision. + return max(revisions) + + +def fetch_grafana_snaps(charm_revision) -> list[Snap]: + """Fetch grafana-agent snaps information.""" + response = requests.get(f"https://raw.githubusercontent.com/canonical/grafana-agent-operator/refs/tags/rev{charm_revision}/src/snap_management.py") + response.raise_for_status() + content = response.text + + snap_name = fetch_var_from_py_file(content, "_grafana_agent_snap_name") + snaps = fetch_var_from_py_file(content, "_grafana_agent_snaps") + + if snap_name and snaps: + result = [] + for (confinement, arch), revision in snaps.items(): + if arch == "amd64": + result.append(Snap( + name=snap_name, + revision=int(revision), + push_channel= "latest/stable" + )) + return result + else: + raise ValueError("Required grafana-agent snap variables not found in the file") + + +def fetch_mysql_snaps(charm_revision) -> list[Snap]: + """Fetch mysql-operator snaps information.""" + resp_revision = requests.get(f"https://raw.githubusercontent.com/canonical/mysql-operator/refs/tags/rev{charm_revision}/snap_revisions.json") + resp_revision.raise_for_status() + resp_name = requests.get(f"https://raw.githubusercontent.com/canonical/mysql-operator/refs/tags/rev{charm_revision}/src/constants.py") + resp_name.raise_for_status() + + snap_revision = resp_revision.json().get("x86_64") + snap_name = fetch_var_from_py_file(resp_name.text, "CHARMED_MYSQL_SNAP_NAME") + + if snap_name and snap_revision: + result = [Snap( + name=snap_name, + revision=int(snap_revision), + push_channel="8.0/edge", + )] + return result + else: + raise ValueError("Required mysql-operator snap variables not found in the file") + + +def fetch_mysql_router_snaps(charm_revision) -> list[Snap]: + """Fetch mysql-router snaps information.""" + response = requests.get(f"https://raw.githubusercontent.com/canonical/mysql-router-operator/refs/tags/rev{charm_revision}/src/snap.py") + response.raise_for_status() + + snap_name = fetch_var_from_py_file(response.text, "_SNAP_NAME") + amd64_rev_number = re.search(r'"x86_64":\s*"(\d+)"', response.text) + revision = amd64_rev_number.group(1) if amd64_rev_number else None + + if snap_name and revision: + result = [Snap( + name=snap_name, + revision=int(revision), + push_channel="8.0/edge", + )] + return result + else: + raise ValueError("Required mysql-router snap variables not found in the file") + + +def fetch_postgresql_snaps(charm_revision) -> list[Snap]: + """Fetch postgresql-operator snaps information.""" + response = requests.get(f"https://raw.githubusercontent.com/canonical/postgresql-operator/refs/tags/rev{charm_revision}/src/constants.py") + response.raise_for_status() + + snap_list = fetch_var_from_py_file(response.text, "SNAP_PACKAGES", False) + + if snap_list: + result = [] + for snap_name, snap_info in snap_list: + result.append(Snap( + name=snap_name, + revision=int(snap_info["revision"]["x86_64"]), + push_channel=snap_info["channel"], + )) + return result + else: + raise ValueError("Required postgresql-operator snap variables not found in the file") + + +def fetch_pgbouncer_snaps(charm_revision) -> list[Snap]: + """Fetch pgbouncer-operator snaps information.""" + response = requests.get(f"https://raw.githubusercontent.com/canonical/pgbouncer-operator/refs/tags/rev{charm_revision}/src/constants.py") + response.raise_for_status() + + snap_list = fetch_var_from_py_file(response.text, "SNAP_PACKAGES", False) + + if snap_list: + result = [] + for snap_name, snap_info in snap_list: + result.append(Snap( + name=snap_name, + revision=int(snap_info["revision"]["x86_64"]), + push_channel=snap_info["channel"], + )) + return result + else: + raise ValueError("Required pgbouncer-operator snap variables not found in the file") + + +def fetch_ubuntu_advantage_snaps() -> list[Snap]: + """Return canonical-livepatch latest revision in default channel (the way the charm deploys it)""" + response = requests.get("https://api.snapcraft.io/v2/snaps/info/canonical-livepatch", headers = {'Snap-Device-Series': '16'}) + response.raise_for_status() + default_channel = response.json()["channel-map"][0] + snap = [Snap( + name="canonical-livepatch", + revision=int(default_channel['revision']), + push_channel=f"{default_channel['channel']['track']}/{default_channel['channel']['risk']}", + )] + return snap + + +SNAP_FETCHERS_BY_CHARM = { + "grafana-agent": fetch_grafana_snaps, + "pgbouncer": fetch_pgbouncer_snaps, + "postgresql": fetch_postgresql_snaps, + "mysql": fetch_mysql_snaps, + "mysql-router": fetch_mysql_router_snaps, + "ubuntu-advantage": fetch_ubuntu_advantage_snaps, +} +SNAPS_YAML_PATH = "releases/latest/snaps.yaml" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("bundle_file_path", type=str) + + bundle_file_path = pathlib.Path(parser.parse_args().bundle_file_path) + old_bundle_data = yaml.safe_load(bundle_file_path.read_text()) + bundle_data = copy.deepcopy(old_bundle_data) + bundle_snaps = set() + bundle_oci_resources = {} + updates_available = False + + # Charm series detection is only supported for top-level and application-level "series" keys + # Other charm series config (e.g. machine-level key) is not supported + # Full list of possible series config (unsupported) can be found under "Charm series" at https://juju.is/docs/olm/bundle + default_series = bundle_data.get("series") + for app in bundle_data["applications"].values(): + channel_map, resources = fetch_charm_info_from_store(app['charm'], app['channel']) + if latest_revision := fetch_latest_charm_revision(channel_map, app.get("series", default_series)): + app["revision"] = latest_revision + else: + raise ValueError( + f"Revision not found for {app['charm']} on {app['channel']} for Ubuntu {app.get('series', default_series)}" + ) + for resource in resources: + if resource["type"] == "oci-image": + response = requests.get(resource["download"]["url"]) + response.raise_for_status() + resource_data = response.json() + + app.setdefault("resources", {}) + app["resources"][resource["name"]] = int(resource["revision"]) + + # Will be added separately, as comments to yaml file + bundle_oci_resources[resource["name"]] = { + "revision": int(resource["revision"]), + "oci-image": f"docker://{resource_data['ImageName']}", + "oci-username": resource_data["Username"], + "oci-password": resource_data["Password"], + } + if app["charm"] in SNAP_FETCHERS_BY_CHARM: + fetcher_func = SNAP_FETCHERS_BY_CHARM[app["charm"]] + if app["charm"] == "ubuntu-advantage": + snaps = fetcher_func() + else: + snaps = fetcher_func(app["revision"]) + bundle_snaps.update(snaps) + + if old_bundle_data != bundle_data: + updates_available = True + with open(bundle_file_path, "w") as file: + yaml_string = yaml.dump(bundle_data) + for oci_resource_name, oci_resource_data in bundle_oci_resources.items(): + comment = ( + f" # oci-image: {oci_resource_data['oci-image']}\n" + f" # oci-password: {oci_resource_data['oci-password']}\n" + f" # oci-username: {oci_resource_data['oci-username']}" + ) + # Find where to insert the comment + insertion_point = f"{oci_resource_name}: {oci_resource_data['revision']}" + yaml_string = yaml_string.replace(insertion_point, f"{insertion_point}\n{comment}") + file.write(yaml_string) + + if len(bundle_snaps) > 0: + snaps_data = {"packages": [dataclasses.asdict(snap) for snap in bundle_snaps]} + try: + old_snaps_data = yaml.safe_load(pathlib.Path(SNAPS_YAML_PATH).read_text()) + except FileNotFoundError: + old_snaps_data = {} + + if old_snaps_data != snaps_data: + updates_available = True + with open(SNAPS_YAML_PATH, "w") as file: + yaml.dump(snaps_data, file) + + github_actions.output["updates_available"] = json.dumps(updates_available)