Use api/v2 to avoid all the login hassle. Alter the existing file.

.github/generate-published-workflow.py

@@ -2,71 +2,130 @@
 
 # /// script
 # dependencies = [
+#   "PyYAML",
 #   "requests",
+#   "rich",
 # ]
 # ///
 
-"""Generate a GitHub workload that runs `tox` on all published charms."""
-
-import base64
-import binascii
-import json
-import os
-import subprocess
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Update a GitHub workload that runs `tox -e unit` on all published charms.
+
+Charms that are not hosted on GitHub are skipped, as well as any charms where
+the source URL could not be found.
+"""
+
+import pathlib
+import urllib.parse
 
 import requests
+import rich.console
+import yaml
 
+console = rich.console.Console()
 
-def _charmcraft_auth_to_macaroon(charmcraft_auth: str):
-    """Decode charmcraft auth into the macaroon."""
-    try:
-        bytes = base64.b64decode(charmcraft_auth.strip().encode())
-        return json.loads(bytes).get('v')
-    except (binascii.Error, json.JSONDecodeError):
-        return None
 
+URL_BASE = 'https://api.charmhub.io/v2/charms/info'
+WORKFLOW = pathlib.Path(__file__).parent / 'workflows' / 'published-charms-tests.yaml'
 
-def macaroon() -> str:
-    """Get the charmhub macaroon."""
-    macaroon = os.environ.get('CHARM_MACAROON')
-    charmcraft_auth = os.environ.get('CHARMCRAFT_AUTH')
-    if not macaroon and charmcraft_auth:
-        macaroon = _charmcraft_auth_to_macaroon(charmcraft_auth)
-    if not macaroon:
-        # Export to stderr because stdout gets a "Login successful" message.
-        out = subprocess.run(
-            ['charmcraft', 'login', '--export', '/dev/fd/2'],
-            text=True,
-            check=True,
-            stderr=subprocess.PIPE,
-        )
-        macaroon = _charmcraft_auth_to_macaroon(out.stderr.splitlines()[-1])
-    if not macaroon:
-        raise ValueError('No charmhub macaroon found')
-    return macaroon.strip()
-
-
-def get_session():
-    session = requests.Session()
-    session.headers['Authorization'] = f'Macaroon {macaroon()}'
-    session.headers['Content-Type'] = 'application/json'
-    return session
+SKIP = {
+    # Handled by db-charm-tests.yaml
+    'postgresql-operator',
+    'postgresql-k8s-operator',
+    'mysql-operator',
+    'mysql-k8s-operator',
+    # Handled by hello-charm-tests.yaml
+    'hello-kubecon',  # Also not in the canonical org, but jnsgruk.
+    'hello-juju-charm',  # Also not in the canonical org, but juju.
+    # Handler by observability-charms-tests.yaml
+    'alertmanager-k8s-operator',
+    'prometheus-k8s-operator',
+    'grafana-k8s-operator',
+}
 
 
 def packages(session: requests.Session):
-    # This works without being logged in, but we might as well re-use the session.
+    """Get the list of published charms from Charmhub."""
+    console.log('Fetching the list of published charms')
     resp = session.get('https://charmhub.io/packages.json')
     return resp.json()['packages']
 
 
-def info(session: requests.Session, charm: str):
-    """Get charm info."""
-    resp = session.get(f'https://api.charmhub.io/v1/charm/{charm}').json()
-    print(resp)
+def get_source_url(charm: str, session: requests.Session):
+    """Get the source URL for a charm."""
+    console.log(f"Looking for a 'source' URL for {charm}")
+    try:
+        source = session.get(f'{URL_BASE}/{charm}?fields=result.links')
+        source.raise_for_status()
+        return source.json()['result']['links']['source'][0]
+    except (requests.HTTPError, KeyError):
+        pass
+    console.log(f"Looking for a 'bugs-url' URL for {charm}")
+    try:
+        source = session.get(f'{URL_BASE}/{charm}?fields=result.bugs-url')
+        source.raise_for_status()
+        return source.json()['result']['bugs-url']
+    except (requests.HTTPError, KeyError):
+        pass
+    # TODO: Can we try anything else?
+    console.log(f'Could not find a source URL for {charm}')
+    return None
+
+
+def url_to_charm_name(url: str):
+    """Get the charm name from a URL."""
+    if not url:
+        return None
+    parsed = urllib.parse.urlparse(url)
+    if parsed.netloc != 'github.com':
+        console.log(f'URL {url} is not a GitHub URL')
+        return None
+    if not parsed.path.startswith('/canonical'):
+        # TODO: Maybe we can include some of these anyway?
+        # 'juju-solutions' and 'charmed-kubernetes' seem viable, for example.
+        console.log(f'URL {url} is not a Canonical charm')
+    try:
+        return urllib.parse.urlparse(url).path.split('/')[2]
+    except IndexError:
+        console.log(f'Could not get charm name from URL {url}')
+        return None
+
+
+def main():
+    """Update the workflow file."""
+    session = requests.Session()
+    charms = (
+        url_to_charm_name(get_source_url(package['name'], session))
+        for package in packages(session)
+    )
+    with WORKFLOW.open('r') as f:
+        workflow = yaml.safe_load(f)
+    workflow['jobs']['charm-tests']['strategy']['matrix']['include'] = [
+        {'charm-repo': f'canonical/{charm}'} for charm in charms if charm and charm not in SKIP
+    ]
+    with WORKFLOW.open('w') as f:
+        yaml.dump(workflow, f)
+    # yaml.safe_load/yaml.dump transforms "on" to "true". I'm not sure how to avoid that.
+    with WORKFLOW.open('r') as f:
+        content = f.read().replace('true:', 'on:')
+    with WORKFLOW.open('w') as f:
+        f.write(content)
+    # TODO: the "Update 'ops' dependency in test charm to latest" run command also gets messed up
+    # and has to get fixed.
 
 
 if __name__ == '__main__':
-    session = get_session()
-    for package in packages(session):
-        info(session, package['name'])
-        break
+    main()