diff --git a/flux_local/git_repo.py b/flux_local/git_repo.py index 043079d4..e56857a1 100644 --- a/flux_local/git_repo.py +++ b/flux_local/git_repo.py @@ -218,6 +218,26 @@ class ResourceVisitor: """ +@dataclass +class DocumentVisitor: + """Invoked when a document is visited so the caller can intercept. + + This is similar to a resource visitor, but it visits the unparsed documents + since they may not have explicit schemas in this project. + """ + + kinds: list[str] + """The resource kinds of documents to visit.""" + + func: Callable[[str, dict[str, Any]], None] + """Function called with the resource and optional content. + + The function arguments are: + - parent: The namespaced name of the Fluxtomization or HelmRelease + - doc: The resource object (e.g. Pod, ConfigMap, HelmRelease, etc) + """ + + @dataclass class MetadataSelector: """A filter for objects to select from the cluster.""" @@ -304,6 +324,9 @@ class ResourceSelector: cluster_policy: MetadataSelector = field(default_factory=MetadataSelector) """ClusterPolicy objects to return.""" + doc_visitor: DocumentVisitor | None = None + """Raw objects to visit.""" + def is_allowed_source(sources: list[Source]) -> Callable[[Kustomization], bool]: """Return true if this Kustomization is from an allowed source.""" @@ -484,20 +507,23 @@ def node_name(ks: Kustomization) -> str: async def build_kustomization( kustomization: Kustomization, cluster_path: Path, - root: Path, - kustomization_selector: MetadataSelector, - helm_release_selector: MetadataSelector, - helm_repo_selector: MetadataSelector, - cluster_policy_selector: MetadataSelector, + selector: ResourceSelector, kustomize_flags: list[str], builder: CachableBuilder, ) -> tuple[Iterable[HelmRepository], Iterable[HelmRelease], Iterable[ClusterPolicy]]: """Build helm objects for the Kustomization.""" + + root: Path = selector.path.root + kustomization_selector: MetadataSelector = selector.kustomization + helm_repo_selector: MetadataSelector = selector.helm_repo + helm_release_selector: MetadataSelector = selector.helm_release + cluster_policy_selector: MetadataSelector = selector.cluster_policy if ( not kustomization_selector.enabled - and not helm_release_selector.enabled and not helm_repo_selector.enabled + and not helm_release_selector.enabled and not cluster_policy_selector.enabled + and not selector.doc_visitor ): return ([], [], []) @@ -519,23 +545,37 @@ async def build_kustomization( ) from err if kustomization_selector.visitor: - if kustomization_selector.visitor: - await kustomization_selector.visitor.func( - cluster_path, - Path(kustomization.path), - kustomization, - cmd, - ) + await kustomization_selector.visitor.func( + cluster_path, + Path(kustomization.path), + kustomization, + cmd, + ) - if ( - not helm_release_selector.enabled - and not helm_repo_selector.enabled - and not cluster_policy_selector.enabled - ): + kinds = [] + if helm_repo_selector.enabled: + kinds.append(HELM_REPO_KIND) + if helm_release_selector.enabled: + kinds.append(HELM_RELEASE_KIND) + if cluster_policy_selector.enabled: + kinds.append(CLUSTER_POLICY_KIND) + if selector.doc_visitor: + kinds.extend(selector.doc_visitor.kinds) + if not kinds: return ([], [], []) - docs = await cmd.grep( - f"kind=^({HELM_REPO_KIND}|{HELM_RELEASE_KIND}|{CLUSTER_POLICY_KIND})$" - ).objects(target_namespace=kustomization.target_namespace) + + regexp = f"kind=^({'|'.join(kinds)})$" + docs = await cmd.grep(regexp).objects( + target_namespace=kustomization.target_namespace + ) + + if selector.doc_visitor: + doc_kinds = set(selector.doc_visitor.kinds) + for doc in docs: + if doc.get("kind") not in doc_kinds: + continue + selector.doc_visitor.func(kustomization.namespaced_name, doc) + return ( filter( helm_repo_selector.predicate, @@ -609,11 +649,7 @@ async def update_kustomization(cluster: Cluster) -> None: build_kustomization( kustomization, Path(cluster.path), - selector.path.root, - selector.kustomization, - selector.helm_release, - selector.helm_repo, - selector.cluster_policy, + selector, options.kustomize_flags, builder, ) diff --git a/flux_local/image.py b/flux_local/image.py new file mode 100644 index 00000000..0b5f9fc6 --- /dev/null +++ b/flux_local/image.py @@ -0,0 +1,74 @@ +"""Helper functions for working with container images.""" + +import logging +from typing import Any + +from . import git_repo, manifest + +_LOGGER = logging.getLogger(__name__) + + +# Object types that may have container images. +KINDS = [ + "Pod", + "Deployment", + "StatefulSet", + "ReplicaSet", + "DaemonSet", + "CronJob", + "Job", + "ReplicationController", +] +IMAGE_KEY = "image" + + +def _extract_images(doc: dict[str, Any]) -> set[str]: + """Extract the image from a Kubernetes object.""" + images: set[str] = set({}) + for key, value in doc.items(): + if key == IMAGE_KEY: + images.add(value) + elif isinstance(value, dict): + images.update(_extract_images(value)) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + images.update(_extract_images(item)) + return images + + +class ImageVisitor: + """Helper that visits container image related objects. + + This tracks the container images used by the kustomizations and HelmReleases + so they can be dumped for further verification. + """ + + def __init__(self) -> None: + """Initialize ImageVisitor.""" + self.images: dict[str, set[str]] = {} + + def repo_visitor(self) -> git_repo.DocumentVisitor: + """Return a git_repo.DocumentVisitor that points to this object.""" + + def add_image(name: str, doc: dict[str, Any]) -> None: + """Visitor function to find relevant images and record them for later inspection. + + Updates the image set with the images found in the document. + """ + images = _extract_images(doc) + if not images: + return + if name in self.images: + self.images[name].update(images) + else: + self.images[name] = set(images) + + return git_repo.DocumentVisitor(kinds=KINDS, func=add_image) + + def update_manifest(self, manifest: manifest.Manifest) -> None: + """Update the manifest with the images found in the repo.""" + for cluster in manifest.clusters: + for kustomization in cluster.kustomizations: + if images := self.images.get(kustomization.namespaced_name): + kustomization.images = list(images) diff --git a/flux_local/manifest.py b/flux_local/manifest.py index a083e173..a26c5998 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -57,7 +57,7 @@ class BaseManifest(BaseModel): _COMPACT_EXCLUDE_FIELDS: dict[str, Any] = {} - def compact_dict(self, exclude: dict[str, Any] | None = None) -> dict[str, Any]: + def compact_dict(self, exclude: dict[str, Any] | None = None, include: dict[str, Any] | None = None) -> dict[str, Any]: """Return a compact dictionary representation of the object. This is similar to `dict()` but with a specific implementation for serializing @@ -65,7 +65,7 @@ def compact_dict(self, exclude: dict[str, Any] | None = None) -> dict[str, Any]: """ if exclude is None: exclude = self._COMPACT_EXCLUDE_FIELDS - return self.dict(exclude=exclude) # type: ignore[arg-type] + return self.dict(exclude=exclude, exclude_unset=True, exclude_none=True, exclude_defaults=True) # type: ignore[arg-type] @classmethod def parse_yaml(cls, content: str) -> "BaseManifest": @@ -293,6 +293,9 @@ class Kustomization(BaseManifest): contents: dict[str, Any] | None = None """Contents of the raw Kustomization document.""" + images: list[str] = Field(default_factory=list) + """The list of images referenced in the kustomization.""" + @classmethod def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization": """Parse a partial Kustomization from a kubernetes resource.""" diff --git a/flux_local/tool/get.py b/flux_local/tool/get.py index 3ab8843c..3d2024d5 100644 --- a/flux_local/tool/get.py +++ b/flux_local/tool/get.py @@ -1,10 +1,11 @@ """Flux-local get action.""" import logging -from argparse import ArgumentParser, _SubParsersAction as SubParsersAction +from argparse import ArgumentParser, BooleanOptionalAction, _SubParsersAction as SubParsersAction from typing import cast, Any +import sys -from flux_local import git_repo +from flux_local import git_repo, image from .format import PrintFormatter, YamlFormatter from . import selector @@ -149,6 +150,13 @@ def register( ), ) selector.add_cluster_selector_flags(args) + args.add_argument( + "--enable-images", + type=str, + default=False, + action=BooleanOptionalAction, + help="Output container images when traversing the cluster", + ) args.add_argument( "--output", "-o", @@ -162,16 +170,42 @@ def register( async def run( # type: ignore[no-untyped-def] self, output: str, + enable_images: bool, **kwargs, # pylint: disable=unused-argument ) -> None: """Async Action implementation.""" query = selector.build_cluster_selector(**kwargs) query.helm_release.enabled = output == "yaml" + + image_visitor: image.ImageVisitor | None = None + if enable_images: + if output != "yaml": + print( + "Flag --enable-images only works with --output yaml", + file=sys.stderr, + ) + return + image_visitor = image.ImageVisitor() + query.doc_visitor = image_visitor.repo_visitor() + manifest = await git_repo.build_manifest( selector=query, options=selector.options(**kwargs) ) if output == "yaml": - YamlFormatter().print([manifest.compact_dict()]) + include: dict[str, Any] | None = None + if image_visitor: + image_visitor.update_manifest(manifest) + include = { + "clusters": { + "__all__": { + "kustomizations": { + "__all__": True, + #"images": True, + } + } + } + } + YamlFormatter().print([manifest.compact_dict(include=include)]) return cols = ["path", "kustomizations"] diff --git a/tests/__snapshots__/test_git_repo.ambr b/tests/__snapshots__/test_git_repo.ambr index d7db3c23..614a4ced 100644 --- a/tests/__snapshots__/test_git_repo.ambr +++ b/tests/__snapshots__/test_git_repo.ambr @@ -39,7 +39,6 @@ 'cluster_policies': list([ dict({ 'name': 'test-allow-policy', - 'namespace': None, }), ]), 'helm_releases': list([ @@ -140,7 +139,6 @@ 'cluster_policies': list([ dict({ 'name': 'test-allow-policy', - 'namespace': None, }), ]), 'helm_releases': list([ @@ -226,7 +224,6 @@ 'cluster_policies': list([ dict({ 'name': 'test-allow-policy', - 'namespace': None, }), ]), 'helm_releases': list([ @@ -352,7 +349,6 @@ 'cluster_policies': list([ dict({ 'name': 'test-allow-policy', - 'namespace': None, }), ]), 'helm_releases': list([ @@ -438,7 +434,6 @@ 'cluster_policies': list([ dict({ 'name': 'test-allow-policy', - 'namespace': None, }), ]), 'helm_releases': list([ @@ -919,7 +914,6 @@ 'cluster_policies': list([ dict({ 'name': 'test-allow-policy', - 'namespace': None, }), ]), 'helm_releases': list([ diff --git a/tests/test_image.py b/tests/test_image.py new file mode 100644 index 00000000..7aa850b3 --- /dev/null +++ b/tests/test_image.py @@ -0,0 +1,41 @@ +"""Tests for image.""" + +from pathlib import Path +from typing import Any + +import pytest +from syrupy.assertion import SnapshotAssertion + +from flux_local.git_repo import build_manifest, ResourceSelector, PathSelector +from flux_local.image import ImageVisitor + +TESTDATA = Path("tests/testdata/cluster8") +CWD = Path.cwd() + + +@pytest.mark.parametrize( + ("test_path", "expected"), + [ + ( + CWD / "tests/testdata/cluster8", + {"flux-system/apps": {"alpine", "busybox"}}, + ), + ( + CWD / "tests/testdata/cluster7", + {}, + ), + ], +) +async def test_image_visitor( + snapshot: SnapshotAssertion, test_path: str, expected: dict[str, Any] +) -> None: + """Tests for building the manifest.""" + + image_visitor = ImageVisitor() + query = ResourceSelector( + path=PathSelector(Path(test_path)), + doc_visitor=image_visitor.repo_visitor(), + ) + await build_manifest(selector=query) + + assert image_visitor.images == expected diff --git a/tests/testdata/cluster8/README.md b/tests/testdata/cluster8/README.md new file mode 100644 index 00000000..2b56dfc2 --- /dev/null +++ b/tests/testdata/cluster8/README.md @@ -0,0 +1,3 @@ +# cluster8 + +This cluster is meant to exercise the container image extraction conde. diff --git a/tests/testdata/cluster8/apps/kustomization.yaml b/tests/testdata/cluster8/apps/kustomization.yaml new file mode 100644 index 00000000..1673237b --- /dev/null +++ b/tests/testdata/cluster8/apps/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - pods.yaml diff --git a/tests/testdata/cluster8/apps/pods.yaml b/tests/testdata/cluster8/apps/pods.yaml new file mode 100644 index 00000000..05d8839a --- /dev/null +++ b/tests/testdata/cluster8/apps/pods.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: mnt-pod + namespace: home +spec: + volumes: + - name: task-pv-storage + persistentVolumeClaim: + claimName: some-pod-datadir + containers: + - name: task-pv-container + image: alpine + volumeMounts: + - mountPath: "/data" + name: task-pv-storage + command: ["/bin/sh"] + args: ["-c", "sleep 500000"] +--- +apiVersion: v1 +kind: Pod +metadata: + name: sleep + namespace: home +spec: + initContainers: + - name: sleep + image: busybox + command: ["/bin/sh"] + args: ["-c", "sleep 500000"] diff --git a/tests/testdata/cluster8/cluster/apps.yaml b/tests/testdata/cluster8/cluster/apps.yaml new file mode 100644 index 00000000..6d63b0e6 --- /dev/null +++ b/tests/testdata/cluster8/cluster/apps.yaml @@ -0,0 +1,18 @@ +--- +# Left on deprecated CRDs for testing +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps + namespace: flux-system +spec: + interval: 10m0s + dependsOn: + - name: infra-configs + sourceRef: + kind: GitRepository + name: flux-system + path: ./tests/testdata/cluster8/apps + prune: true + wait: true + timeout: 5m0s diff --git a/tests/testdata/cluster8/cluster/flux-system/gotk-sync.yaml b/tests/testdata/cluster8/cluster/flux-system/gotk-sync.yaml new file mode 100644 index 00000000..43cd28cb --- /dev/null +++ b/tests/testdata/cluster8/cluster/flux-system/gotk-sync.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: flux-system + namespace: flux-system +spec: + interval: 1m0s + ref: + branch: main + secretRef: + name: flux-system + url: ssh://git@github.com/allenporter/flux-local +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: flux-system + namespace: flux-system +spec: + interval: 10m0s + path: ./tests/testdata/cluster8/cluster/ + prune: true + sourceRef: + kind: GitRepository + name: flux-system diff --git a/tests/testdata/cluster8/cluster/flux-system/kustomization.yaml b/tests/testdata/cluster8/cluster/flux-system/kustomization.yaml new file mode 100644 index 00000000..197efd8a --- /dev/null +++ b/tests/testdata/cluster8/cluster/flux-system/kustomization.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - gotk-sync.yaml diff --git a/tests/tool/__snapshots__/test_get_cluster.ambr b/tests/tool/__snapshots__/test_get_cluster.ambr index d42a9989..fc0a4bac 100644 --- a/tests/tool/__snapshots__/test_get_cluster.ambr +++ b/tests/tool/__snapshots__/test_get_cluster.ambr @@ -62,6 +62,27 @@ ''' # --- +# name: test_get_cluster[yaml-cluster8-images] + ''' + --- + clusters: + - path: tests/testdata/cluster8 + kustomizations: + - name: apps + namespace: flux-system + path: tests/testdata/cluster8/apps + helm_repos: [] + helm_releases: [] + cluster_policies: [] + - name: flux-system + namespace: flux-system + path: tests/testdata/cluster8/cluster + helm_repos: [] + helm_releases: [] + cluster_policies: [] + + ''' +# --- # name: test_get_cluster[yaml] ''' --- @@ -105,7 +126,6 @@ helm_releases: [] cluster_policies: - name: test-allow-policy - namespace: null - name: infra-controllers namespace: flux-system path: tests/testdata/cluster/infrastructure/controllers diff --git a/tests/tool/test_get_cluster.py b/tests/tool/test_get_cluster.py index 8f9f8b89..5ebeb9f0 100644 --- a/tests/tool/test_get_cluster.py +++ b/tests/tool/test_get_cluster.py @@ -26,7 +26,8 @@ (["--path", "tests/testdata/cluster6"]), (["--path", "tests/testdata/cluster7"]), (["--all-namespaces", "--path", "tests/testdata/cluster/"]), - (["--path", "tests/testdata/cluster/", "-o", "yaml"]), + (["--path", "tests/testdata/cluster", "-o", "yaml"]), + (["--path", "tests/testdata/cluster8", "-o", "yaml"]), ], ids=[ "cluster", @@ -39,6 +40,7 @@ "cluster7", "all-namespaces", "yaml", + "yaml-cluster8-images" ], ) async def test_get_cluster(args: list[str], snapshot: SnapshotAssertion) -> None: