From 09bcca9e4e5dbd753cf1a912b86f8e601c2b819b Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Thu, 21 Dec 2023 07:23:52 -0800 Subject: [PATCH] Add HelmRelease images to cluster manifest (#445) Follow on from #443 by also adding HelmRelease images to the manifest. This also contains cleanup of other unused code from the initial PR adding container images. Issue #434 Example usage ``` $ flux-local get cluster --path tests/testdata/cluster/ -o yaml --enable-images --- clusters: - path: tests/testdata/cluster kustomizations: - name: apps namespace: flux-system path: tests/testdata/cluster/apps/prod helm_repos: [] helm_releases: - name: podinfo namespace: podinfo chart: name: podinfo repo_name: podinfo repo_namespace: flux-system images: - public.ecr.aws/docker/library/redis:7.0.6 - ghcr.io/stefanprodan/podinfo:6.3.2 cluster_policies: [] - name: flux-system namespace: flux-system path: tests/testdata/cluster/clusters/prod helm_repos: [] helm_releases: [] cluster_policies: [] - name: infra-configs namespace: flux-system path: tests/testdata/cluster/infrastructure/configs helm_repos: - name: bitnami namespace: flux-system url: https://charts.bitnami.com/bitnami repo_type: default - name: podinfo namespace: flux-system url: oci://ghcr.io/stefanprodan/charts repo_type: oci - name: weave-charts namespace: flux-system url: oci://ghcr.io/weaveworks/charts repo_type: oci helm_releases: [] cluster_policies: - name: test-allow-policy - name: infra-controllers namespace: flux-system path: tests/testdata/cluster/infrastructure/controllers helm_repos: [] helm_releases: - name: metallb namespace: metallb chart: name: metallb repo_name: bitnami repo_namespace: flux-system images: - docker.io/bitnami/metallb-controller:0.13.7-debian-11-r29 - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28 - name: weave-gitops namespace: flux-system chart: name: weave-gitops repo_name: weave-charts repo_namespace: flux-system images: - ghcr.io/weaveworks/wego-app:v0.24.0 cluster_policies: [] ``` --- flux_local/git_repo.py | 2 +- flux_local/image.py | 1 + flux_local/manifest.py | 14 ++- flux_local/tool/get.py | 39 ++++--- flux_local/tool/visitor.py | 39 ++++++- .../tool/__snapshots__/test_get_cluster.ambr | 108 +++++++++++++++++- tests/tool/test_get_cluster.py | 6 +- 7 files changed, 183 insertions(+), 26 deletions(-) diff --git a/flux_local/git_repo.py b/flux_local/git_repo.py index e56857a1..3ef5902b 100644 --- a/flux_local/git_repo.py +++ b/flux_local/git_repo.py @@ -568,7 +568,7 @@ async def build_kustomization( docs = await cmd.grep(regexp).objects( target_namespace=kustomization.target_namespace ) - + if selector.doc_visitor: doc_kinds = set(selector.doc_visitor.kinds) for doc in docs: diff --git a/flux_local/image.py b/flux_local/image.py index 0b5f9fc6..65d647f6 100644 --- a/flux_local/image.py +++ b/flux_local/image.py @@ -72,3 +72,4 @@ def update_manifest(self, manifest: manifest.Manifest) -> None: for kustomization in cluster.kustomizations: if images := self.images.get(kustomization.namespaced_name): kustomization.images = list(images) + kustomization.images.sort() diff --git a/flux_local/manifest.py b/flux_local/manifest.py index a26c5998..ce1595e5 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -57,7 +57,7 @@ class BaseManifest(BaseModel): _COMPACT_EXCLUDE_FIELDS: dict[str, Any] = {} - def compact_dict(self, exclude: dict[str, Any] | None = None, include: dict[str, Any] | None = None) -> dict[str, Any]: + def compact_dict(self, exclude: dict[str, Any] | None = None) -> dict[str, Any]: """Return a compact dictionary representation of the object. This is similar to `dict()` but with a specific implementation for serializing @@ -143,6 +143,9 @@ class HelmRelease(BaseManifest): values: Optional[dict[str, Any]] = None """The values to install in the chart.""" + images: list[str] = Field(default_factory=list) + """The list of images referenced in the HelmRelease.""" + @classmethod def parse_doc(cls, doc: dict[str, Any]) -> "HelmRelease": """Parse a HelmRelease from a kubernetes resource object.""" @@ -171,6 +174,11 @@ def repo_name(self) -> str: """Identifier for the HelmRepository identified in the HelmChart.""" return f"{self.chart.repo_namespace}-{self.chart.repo_name}" + @property + def namespaced_name(self) -> str: + """Return the namespace and name concatenated as an id.""" + return f"{self.namespace}/{self.name}" + _COMPACT_EXCLUDE_FIELDS = { "values": True, "chart": HelmChart._COMPACT_EXCLUDE_FIELDS, @@ -329,9 +337,9 @@ def id_name(self) -> str: return f"{self.path}" @property - def namespaced_name(self, sep: str = "/") -> str: + def namespaced_name(self) -> str: """Return the namespace and name concatenated as an id.""" - return f"{self.namespace}{sep}{self.name}" + return f"{self.namespace}/{self.name}" _COMPACT_EXCLUDE_FIELDS = { "helm_releases": { diff --git a/flux_local/tool/get.py b/flux_local/tool/get.py index 3d2024d5..cab92628 100644 --- a/flux_local/tool/get.py +++ b/flux_local/tool/get.py @@ -1,14 +1,21 @@ """Flux-local get action.""" import logging -from argparse import ArgumentParser, BooleanOptionalAction, _SubParsersAction as SubParsersAction +from argparse import ( + ArgumentParser, + BooleanOptionalAction, + _SubParsersAction as SubParsersAction, +) from typing import cast, Any import sys +import pathlib +import tempfile -from flux_local import git_repo, image +from flux_local import git_repo, image, helm from .format import PrintFormatter, YamlFormatter from . import selector +from .visitor import HelmVisitor, ImageOutput _LOGGER = logging.getLogger(__name__) @@ -178,6 +185,7 @@ async def run( # type: ignore[no-untyped-def] query.helm_release.enabled = output == "yaml" image_visitor: image.ImageVisitor | None = None + helm_content: ImageOutput | None = None if enable_images: if output != "yaml": print( @@ -188,24 +196,27 @@ async def run( # type: ignore[no-untyped-def] image_visitor = image.ImageVisitor() query.doc_visitor = image_visitor.repo_visitor() + helm_content = ImageOutput() + helm_visitor = HelmVisitor() + query.helm_repo.visitor = helm_visitor.repo_visitor() + query.helm_release.visitor = helm_visitor.release_visitor() + manifest = await git_repo.build_manifest( selector=query, options=selector.options(**kwargs) ) if output == "yaml": - include: dict[str, Any] | None = None if image_visitor: image_visitor.update_manifest(manifest) - include = { - "clusters": { - "__all__": { - "kustomizations": { - "__all__": True, - #"images": True, - } - } - } - } - YamlFormatter().print([manifest.compact_dict(include=include)]) + if helm_content: + with tempfile.TemporaryDirectory() as helm_cache_dir: + await helm_visitor.inflate( + pathlib.Path(helm_cache_dir), + helm_content.visitor(), + helm.Options(), + ) + helm_content.update_manifest(manifest) + + YamlFormatter().print([manifest.compact_dict()]) return cols = ["path", "kustomizations"] diff --git a/flux_local/tool/visitor.py b/flux_local/tool/visitor.py index 9dc01f4b..03214ced 100644 --- a/flux_local/tool/visitor.py +++ b/flux_local/tool/visitor.py @@ -9,7 +9,7 @@ import yaml from typing import Any -from flux_local import git_repo +from flux_local import git_repo, image from flux_local.helm import Helm, Options from flux_local.kustomize import Kustomize from flux_local.manifest import ( @@ -17,6 +17,7 @@ Kustomization, HelmRepository, ClusterPolicy, + Manifest, ) @@ -153,6 +154,42 @@ def strip_attrs(metadata: dict[str, Any], strip_attributes: list[str]) -> None: break +class ImageOutput(ResourceOutput): + """Resource visitor that builds outputs for objects within the kustomization.""" + + def __init__(self) -> None: + """Initialize ObjectOutput.""" + # Map of kustomizations to the map of built objects as lines of the yaml string + self.content: dict[ResourceKey, dict[ResourceKey, list[str]]] = {} + self.image_visitor = image.ImageVisitor() + self.repo_visitor = self.image_visitor.repo_visitor() + + async def call_async( + self, + cluster_path: pathlib.Path, + kustomization_path: pathlib.Path, + doc: ResourceType, + cmd: Kustomize | None, + ) -> None: + """Visitor function invoked to build and record resource objects.""" + if cmd and isinstance(doc, HelmRelease): + objects = await cmd.objects() + for obj in objects: + if obj.get("kind") in self.repo_visitor.kinds: + self.repo_visitor.func(doc.namespaced_name, obj) + + def update_manifest(self, manifest: Manifest) -> None: + """Update the manifest with the images found in the repo.""" + for cluster in manifest.clusters: + for kustomization in cluster.kustomizations: + for helm_release in kustomization.helm_releases: + if images := self.image_visitor.images.get( + helm_release.namespaced_name + ): + helm_release.images = list(images) + helm_release.images.sort() + + class ObjectOutput(ResourceOutput): """Resource visitor that builds outputs for objects within the kustomization.""" diff --git a/tests/tool/__snapshots__/test_get_cluster.ambr b/tests/tool/__snapshots__/test_get_cluster.ambr index fc0a4bac..38d77fa1 100644 --- a/tests/tool/__snapshots__/test_get_cluster.ambr +++ b/tests/tool/__snapshots__/test_get_cluster.ambr @@ -62,28 +62,79 @@ ''' # --- -# name: test_get_cluster[yaml-cluster8-images] +# name: test_get_cluster[yaml-cluster-images] ''' --- clusters: - - path: tests/testdata/cluster8 + - path: tests/testdata/cluster kustomizations: - name: apps namespace: flux-system - path: tests/testdata/cluster8/apps + path: tests/testdata/cluster/apps/prod helm_repos: [] - helm_releases: [] + helm_releases: + - name: podinfo + namespace: podinfo + chart: + name: podinfo + repo_name: podinfo + repo_namespace: flux-system + images: + - ghcr.io/stefanprodan/podinfo:6.3.2 + - public.ecr.aws/docker/library/redis:7.0.6 cluster_policies: [] - name: flux-system namespace: flux-system - path: tests/testdata/cluster8/cluster + path: tests/testdata/cluster/clusters/prod helm_repos: [] helm_releases: [] cluster_policies: [] + - name: infra-configs + namespace: flux-system + path: tests/testdata/cluster/infrastructure/configs + helm_repos: + - name: bitnami + namespace: flux-system + url: https://charts.bitnami.com/bitnami + repo_type: default + - name: podinfo + namespace: flux-system + url: oci://ghcr.io/stefanprodan/charts + repo_type: oci + - name: weave-charts + namespace: flux-system + url: oci://ghcr.io/weaveworks/charts + repo_type: oci + helm_releases: [] + cluster_policies: + - name: test-allow-policy + - name: infra-controllers + namespace: flux-system + path: tests/testdata/cluster/infrastructure/controllers + helm_repos: [] + helm_releases: + - name: metallb + namespace: metallb + chart: + name: metallb + repo_name: bitnami + repo_namespace: flux-system + images: + - docker.io/bitnami/metallb-controller:0.13.7-debian-11-r29 + - docker.io/bitnami/metallb-speaker:0.13.7-debian-11-r28 + - name: weave-gitops + namespace: flux-system + chart: + name: weave-gitops + repo_name: weave-charts + repo_namespace: flux-system + images: + - ghcr.io/weaveworks/wego-app:v0.24.0 + cluster_policies: [] ''' # --- -# name: test_get_cluster[yaml] +# name: test_get_cluster[yaml-cluster-no-images] ''' --- clusters: @@ -147,3 +198,48 @@ ''' # --- +# name: test_get_cluster[yaml-cluster8-images] + ''' + --- + clusters: + - path: tests/testdata/cluster8 + kustomizations: + - name: apps + namespace: flux-system + path: tests/testdata/cluster8/apps + helm_repos: [] + helm_releases: [] + cluster_policies: [] + images: + - alpine + - busybox + - name: flux-system + namespace: flux-system + path: tests/testdata/cluster8/cluster + helm_repos: [] + helm_releases: [] + cluster_policies: [] + + ''' +# --- +# name: test_get_cluster[yaml-cluster8-no-images] + ''' + --- + clusters: + - path: tests/testdata/cluster8 + kustomizations: + - name: apps + namespace: flux-system + path: tests/testdata/cluster8/apps + helm_repos: [] + helm_releases: [] + cluster_policies: [] + - name: flux-system + namespace: flux-system + path: tests/testdata/cluster8/cluster + helm_repos: [] + helm_releases: [] + cluster_policies: [] + + ''' +# --- diff --git a/tests/tool/test_get_cluster.py b/tests/tool/test_get_cluster.py index 5ebeb9f0..a4d18dc4 100644 --- a/tests/tool/test_get_cluster.py +++ b/tests/tool/test_get_cluster.py @@ -27,7 +27,9 @@ (["--path", "tests/testdata/cluster7"]), (["--all-namespaces", "--path", "tests/testdata/cluster/"]), (["--path", "tests/testdata/cluster", "-o", "yaml"]), + (["--path", "tests/testdata/cluster", "-o", "yaml", "--enable-images"]), (["--path", "tests/testdata/cluster8", "-o", "yaml"]), + (["--path", "tests/testdata/cluster8", "-o", "yaml", "--enable-images"]), ], ids=[ "cluster", @@ -39,7 +41,9 @@ "cluster6", "cluster7", "all-namespaces", - "yaml", + "yaml-cluster-no-images", + "yaml-cluster-images", + "yaml-cluster8-no-images", "yaml-cluster8-images" ], )