From 5db447d52bcd8713b89c1dc50ff2de71e90645c1 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Sun, 19 Feb 2023 11:01:49 -0800 Subject: [PATCH] Add a flux-local get cluster command to output flux cluster details (#65) Add a flux-local get cluster command to output flux cluster details which can be used to generate the manifest. --- .pre-commit-config.yaml | 4 +- .yaml-lint.yaml | 2 +- flux_local/git_repo.py | 51 +++++-- flux_local/helm.py | 4 +- flux_local/manifest.py | 127 ++++++++++-------- flux_local/tool/diff.py | 2 +- flux_local/tool/flux_local.py | 10 +- flux_local/tool/format.py | 52 ++++++- flux_local/tool/get.py | 106 +++++++++++---- flux_local/tool/manifest.py | 18 --- flux_local/tool/selector.py | 18 +++ tests/test_git_repo.py | 56 +++++++- tests/test_helm.py | 4 +- tests/test_manifest.py | 57 ++++++-- tests/{ => tool}/test_flux_local.py | 2 +- tests/tool/test_format.py | 109 +++++++++++++++ .../flux_local => tool/testdata}/build.yaml | 0 .../testdata}/build_helm.yaml | 0 .../flux_local => tool/testdata}/diff_hr.yaml | 0 .../testdata}/diff_hr_all.yaml | 0 .../testdata}/diff_hr_all_namespaces.yaml | 0 .../testdata}/diff_hr_not_found.yaml | 0 .../diff_hr_not_found_all_namespaces.yaml | 0 .../diff_hr_not_found_namespace.yaml | 0 .../flux_local => tool/testdata}/diff_ks.yaml | 0 tests/tool/testdata/get_cluster.yaml | 8 ++ tests/tool/testdata/get_cluster_all.yaml | 9 ++ .../testdata/get_cluster_yaml.yaml} | 7 +- .../flux_local => tool/testdata}/get_hr.yaml | 0 .../testdata}/get_hr_name.yaml | 0 .../testdata}/get_hr_namespace.yaml | 0 .../flux_local => tool/testdata}/get_ks.yaml | 0 32 files changed, 498 insertions(+), 148 deletions(-) delete mode 100644 flux_local/tool/manifest.py rename tests/{ => tool}/test_flux_local.py (92%) create mode 100644 tests/tool/test_format.py rename tests/{testdata/flux_local => tool/testdata}/build.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/build_helm.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_hr.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_hr_all.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_hr_all_namespaces.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_hr_not_found.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_hr_not_found_all_namespaces.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_hr_not_found_namespace.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/diff_ks.yaml (100%) create mode 100644 tests/tool/testdata/get_cluster.yaml create mode 100644 tests/tool/testdata/get_cluster_all.yaml rename tests/{testdata/flux_local/manifest.yaml => tool/testdata/get_cluster_yaml.yaml} (94%) rename tests/{testdata/flux_local => tool/testdata}/get_hr.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/get_hr_name.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/get_hr_namespace.yaml (100%) rename tests/{testdata/flux_local => tool/testdata}/get_ks.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 666101e9..1a2d9c4a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: rev: v4.0.1 hooks: - id: trailing-whitespace - exclude: '^tests/testdata/flux_local/.*\.yaml$' + exclude: '^tests/tool/testdata/.*\.yaml$' - id: end-of-file-fixer - id: check-yaml args: @@ -18,7 +18,7 @@ repos: rev: v1.29.0 hooks: - id: yamllint - exclude: '^tests/testdata/flux_local/.*\.yaml$' + exclude: '^tests/tool/testdata/.*\.yaml$' args: - -c - ".yaml-lint.yaml" diff --git a/.yaml-lint.yaml b/.yaml-lint.yaml index 9affa809..77696a87 100644 --- a/.yaml-lint.yaml +++ b/.yaml-lint.yaml @@ -1,7 +1,7 @@ --- ignore: | venv - tests/testdata/flux_local/ + tests/tool/testdata/ extends: default rules: truthy: diff --git a/flux_local/git_repo.py b/flux_local/git_repo.py index f76cede1..94d99dfc 100644 --- a/flux_local/git_repo.py +++ b/flux_local/git_repo.py @@ -127,6 +127,9 @@ def process_path(self) -> Path: class MetadataSelector: """A filter for objects to select from the cluster.""" + enabled: bool = True + """If true, this selector may return objects.""" + name: str | None = None """Resources returned will match this name.""" @@ -134,10 +137,12 @@ class MetadataSelector: """Resources returned will be from this namespace.""" @property - def predicate(self) -> Callable[[Kustomization | HelmRelease], bool]: + def predicate(self) -> Callable[[Kustomization | HelmRelease | Cluster], bool]: """A predicate that selects Kustomization objects.""" - def predicate(obj: Kustomization | HelmRelease) -> bool: + def predicate(obj: Kustomization | HelmRelease | Cluster) -> bool: + if not self.enabled: + return False if self.name and obj.name != self.name: return False if self.namespace and obj.namespace != self.namespace: @@ -147,6 +152,11 @@ def predicate(obj: Kustomization | HelmRelease) -> bool: return predicate +def cluster_metadata_selector() -> MetadataSelector: + """Create a new MetadataSelector for Kustomizations.""" + return MetadataSelector(name=CLUSTER_KUSTOMIZE_NAME, namespace=DEFAULT_NAMESPACE) + + def ks_metadata_selector() -> MetadataSelector: """Create a new MetadataSelector for Kustomizations.""" return MetadataSelector(namespace=DEFAULT_NAMESPACE) @@ -164,21 +174,31 @@ class ResourceSelector: path: PathSelector = field(default_factory=PathSelector) """Path to find a repo of local flux Kustomization objects""" + cluster: MetadataSelector = field(default_factory=cluster_metadata_selector) + """Cluster names to return, or all if empty.""" + kustomization: MetadataSelector = field(default_factory=ks_metadata_selector) """Kustomization names to return, or all if empty.""" helm_release: MetadataSelector = field(default_factory=MetadataSelector) -async def get_clusters(path: Path) -> list[Cluster]: +async def get_clusters(path: Path, selector: MetadataSelector) -> list[Cluster]: """Load Cluster objects from the specified path.""" cmd = kustomize.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}", path).grep( - f"metadata.name={CLUSTER_KUSTOMIZE_NAME}" + f"metadata.name={selector.name}", ) docs = await cmd.objects() - return [ - Cluster.from_doc(doc) for doc in docs if CLUSTER_KUSTOMIZE_DOMAIN_FILTER(doc) - ] + return list( + filter( + selector.predicate, + [ + Cluster.parse_doc(doc) + for doc in docs + if CLUSTER_KUSTOMIZE_DOMAIN_FILTER(doc) + ], + ) + ) async def get_cluster_kustomizations(path: Path) -> list[Kustomization]: @@ -189,7 +209,7 @@ async def get_cluster_kustomizations(path: Path) -> list[Kustomization]: ) docs = await cmd.objects() return [ - Kustomization.from_doc(doc) + Kustomization.parse_doc(doc) for doc in docs if CLUSTER_KUSTOMIZE_DOMAIN_FILTER(doc) ] @@ -215,18 +235,21 @@ async def build_manifest( selector.path = PathSelector(path=path) _LOGGER.debug("Processing cluster with selector %s", selector) - clusters = await get_clusters(selector.path.process_path) + if not selector.cluster.enabled: + return Manifest(clusters=[]) + + clusters = await get_clusters(selector.path.process_path, selector.cluster) if len(clusters) > 0: for cluster in clusters: _LOGGER.debug("Processing cluster: %s", cluster.path) cluster.kustomizations = await get_cluster_kustomizations( - selector.path.root / cluster.path.lstrip("./") + selector.path.root / cluster.path ) elif selector.path.path: _LOGGER.debug("No clusters found; Processing as a Kustomization: %s", selector) # The argument path may be a Kustomization inside a cluster. Create a synthetic # cluster with any found Kustomizations - cluster = Cluster(name="", path="") + cluster = Cluster(name="", namespace="", path="") objects = await get_kustomizations(selector.path.path) if objects: cluster.kustomizations = [ @@ -238,18 +261,20 @@ async def build_manifest( cluster.kustomizations = list( filter(selector.kustomization.predicate, cluster.kustomizations) ) + if not selector.helm_release.enabled: + continue for kustomization in cluster.kustomizations: _LOGGER.debug("Processing kustomization: %s", kustomization.path) cmd = kustomize.build(selector.path.root / kustomization.path) kustomization.helm_repos = [ - HelmRepository.from_doc(doc) + HelmRepository.parse_doc(doc) for doc in await cmd.grep(f"kind=^{HELM_REPO_KIND}$").objects() ] kustomization.helm_releases = list( filter( selector.helm_release.predicate, [ - HelmRelease.from_doc(doc) + HelmRelease.parse_doc(doc) for doc in await cmd.grep( f"kind=^{HELM_RELEASE_KIND}$" ).objects() diff --git a/flux_local/helm.py b/flux_local/helm.py index b32a201a..40e446dc 100644 --- a/flux_local/helm.py +++ b/flux_local/helm.py @@ -14,7 +14,7 @@ repos = await kustomize.grep("kind=^HelmRepository$").objects() helm = Helm("/tmp/path/helm", "/tmp/path/cache") for repo in repos: - helm.add_repo(HelmRepository.from_doc(repo)) + helm.add_repo(HelmRepository.parse_doc(repo)) await helm.update() ``` @@ -26,7 +26,7 @@ if not len(releases) == 1: raise ValueError("Expected only one HelmRelease") tmpl = helm.template( - HelmRelease.from_doc(releases[0]), + HelmRelease.parse_doc(releases[0]), releases[0]["spec"].get("values")) objects = await tmpl.objects() for object in objects: diff --git a/flux_local/manifest.py b/flux_local/manifest.py index eb5d748e..c4f2b3c7 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -40,7 +40,32 @@ def _check_version(doc: dict[str, Any], version: str) -> None: raise ValueError(f"Invalid object expected '{version}': {doc}") -class HelmChart(BaseModel): +class BaseManifest(BaseModel): + """Base class for all manifest objects.""" + + _COMPACT_EXCLUDE_FIELDS: dict[str, Any] = {} + + def compact_dict(self) -> dict[str, Any]: + """Return a compact dictionary representation of the object. + + This is similar to `dict()` but with a specific implementation for serializing + with variable fields removed. + """ + return self.dict(exclude=self._COMPACT_EXCLUDE_FIELDS) # type: ignore[arg-type] + + @classmethod + def parse_yaml(cls, content: str) -> "BaseManifest": + """Parse a serialized manifest.""" + doc = next(yaml.load_all(content, Loader=yaml.Loader), None) + return cls.parse_obj(doc) + + def yaml(self) -> str: + """Return a YAML string representation of compact_dict.""" + data = self.compact_dict() + return cast(str, yaml.dump(data, sort_keys=False, explicit_start=True)) + + +class HelmChart(BaseManifest): """A representation of an instantiation of a chart for a HelmRelease.""" name: str @@ -56,7 +81,7 @@ class HelmChart(BaseModel): """The namespace of the HelmRepository.""" @classmethod - def from_doc(cls, doc: dict[str, Any]) -> "HelmChart": + def parse_doc(cls, doc: dict[str, Any]) -> "HelmChart": """Parse a HelmChart from a HelmRelease resource object.""" _check_version(doc, HELM_RELEASE_DOMAIN) if not (spec := doc.get("spec")): @@ -85,8 +110,10 @@ def chart_name(self) -> str: """Identifier for the HelmChart.""" return f"{self.repo_namespace}-{self.repo_name}/{self.name}" + _COMPACT_EXCLUDE_FIELDS = {"version": True} -class HelmRelease(BaseModel): + +class HelmRelease(BaseManifest): """A representation of a Flux HelmRelease.""" name: str @@ -102,7 +129,7 @@ class HelmRelease(BaseModel): """The values to install in the chart.""" @classmethod - def from_doc(cls, doc: dict[str, Any]) -> "HelmRelease": + def parse_doc(cls, doc: dict[str, Any]) -> "HelmRelease": """Parse a HelmRelease from a kubernetes resource object.""" _check_version(doc, HELM_RELEASE_DOMAIN) if not (metadata := doc.get("metadata")): @@ -111,7 +138,7 @@ def from_doc(cls, doc: dict[str, Any]) -> "HelmRelease": raise ValueError(f"Invalid {cls} missing metadata.name: {doc}") if not (namespace := metadata.get("namespace")): raise ValueError(f"Invalid {cls} missing metadata.namespace: {doc}") - chart = HelmChart.from_doc(doc) + chart = HelmChart.parse_doc(doc) return cls( name=name, namespace=namespace, @@ -124,8 +151,13 @@ def release_name(self) -> str: """Identifier for the HelmRelease.""" return f"{self.namespace}-{self.name}" + _COMPACT_EXCLUDE_FIELDS = { + "values": True, + "chart": HelmChart._COMPACT_EXCLUDE_FIELDS, + } + -class HelmRepository(BaseModel): +class HelmRepository(BaseManifest): """A representation of a flux HelmRepository.""" name: str @@ -138,7 +170,7 @@ class HelmRepository(BaseModel): """The URL to the repository of helm charts.""" @classmethod - def from_doc(cls, doc: dict[str, Any]) -> "HelmRepository": + def parse_doc(cls, doc: dict[str, Any]) -> "HelmRepository": """Parse a HelmRepository from a kubernetes resource.""" _check_version(doc, HELM_REPO_DOMAIN) if not (metadata := doc.get("metadata")): @@ -159,7 +191,7 @@ def repo_name(self) -> str: return f"{self.namespace}-{self.name}" -class Kustomization(BaseModel): +class Kustomization(BaseManifest): """A Kustomization is a set of declared cluster artifacts. This represents a flux Kustomization that points to a path that @@ -183,7 +215,7 @@ class Kustomization(BaseModel): """The set of HelmRelease represented in this kustomization.""" @classmethod - def from_doc(cls, doc: dict[str, Any]) -> "Kustomization": + def parse_doc(cls, doc: dict[str, Any]) -> "Kustomization": """Parse a partial Kustomization from a kubernetes resource.""" _check_version(doc, CLUSTER_KUSTOMIZE_DOMAIN) if not (metadata := doc.get("metadata")): @@ -203,8 +235,14 @@ def id_name(self) -> str: """Identifier for the Kustomization in tests""" return f"{self.path}" + _COMPACT_EXCLUDE_FIELDS = { + "helm_releases": { + "__all__": HelmRelease._COMPACT_EXCLUDE_FIELDS, + } + } -class Cluster(BaseModel): + +class Cluster(BaseManifest): """A set of nodes that run containerized applications. Many flux git repos will only have a single flux cluster, though @@ -214,6 +252,9 @@ class Cluster(BaseModel): name: str """The name of the cluster.""" + namespace: str + """The namespace of the cluster.""" + path: str """The local git repo path to the Kustomization objects for the cluster.""" @@ -221,18 +262,20 @@ class Cluster(BaseModel): """A list of flux Kustomizations for the cluster.""" @classmethod - def from_doc(cls, doc: dict[str, Any]) -> "Cluster": + def parse_doc(cls, doc: dict[str, Any]) -> "Cluster": """Parse a partial Kustomization from a kubernetes resource.""" _check_version(doc, CLUSTER_KUSTOMIZE_DOMAIN) if not (metadata := doc.get("metadata")): raise ValueError(f"Invalid {cls} missing metadata: {doc}") if not (name := metadata.get("name")): raise ValueError(f"Invalid {cls} missing metadata.name: {doc}") + if not (namespace := metadata.get("namespace")): + raise ValueError(f"Invalid {cls} missing metadata.namespace: {doc}") if not (spec := doc.get("spec")): raise ValueError(f"Invalid {cls} missing spec: {doc}") if not (path := spec.get("path")): raise ValueError(f"Invalid {cls} missing spec.path: {doc}") - return Cluster(name=name, path=path) + return Cluster(name=name, namespace=namespace, path=path) @property def id_name(self) -> str: @@ -257,50 +300,24 @@ def helm_releases(self) -> list[HelmRelease]: for release in kustomization.helm_releases ] + _COMPACT_EXCLUDE_FIELDS = { + "kustomizations": { + "__all__": Kustomization._COMPACT_EXCLUDE_FIELDS, + } + } + -class Manifest(BaseModel): +class Manifest(BaseManifest): """Holds information about cluster and applications contained in a repo.""" clusters: list[Cluster] """A list of Clusters represented in the repo.""" - @staticmethod - def parse_yaml(content: str) -> "Manifest": - """Parse a serialized manifest.""" - doc = next(yaml.load_all(content, Loader=yaml.Loader), None) - return Manifest.parse_obj(doc) - - def yaml(self) -> str: - """Serialize the manifest as a yaml file. - - The contents of the cluster will be compacted to remove values that - exist in the live cluster but do not make sense to be persisted in the - manifest on disk. - """ - data = self.dict( - exclude={ - "clusters": { - "__all__": { - "kustomizations": { - "__all__": { - "helm_releases": { - "__all__": { - "values": True, - "chart": { - "version": True, - }, - }, - }, - }, - }, - }, - }, - } - ) - return cast( - str, - yaml.dump(data, sort_keys=False, explicit_start=True), - ) + _COMPACT_EXCLUDE_FIELDS = { + "clusters": { + "__all__": Cluster._COMPACT_EXCLUDE_FIELDS, + } + } class ManifestException(Exception): @@ -308,10 +325,14 @@ class ManifestException(Exception): async def read_manifest(manifest_path: Path) -> Manifest: - """Return the contents of a serialized manifest file.""" + """Return the contents of a serialized manifest file. + + A manifest file is typically created by `flux-local get cluster -o yaml` or + similar command. + """ async with aiofiles.open(str(manifest_path)) as manifest_file: content = await manifest_file.read() - return Manifest.parse_yaml(content) + return cast(Manifest, Manifest.parse_yaml(content)) async def write_manifest(manifest_path: Path, manifest: Manifest) -> None: @@ -323,9 +344,9 @@ async def write_manifest(manifest_path: Path, manifest: Manifest) -> None: async def update_manifest(manifest_path: Path, manifest: Manifest) -> None: """Write the specified manifest only if changed.""" + new_content = manifest.yaml() async with aiofiles.open(str(manifest_path)) as manifest_file: content = await manifest_file.read() - new_content = manifest.yaml() if content == new_content: return async with aiofiles.open(str(manifest_path), mode="w") as manifest_file: diff --git a/flux_local/tool/diff.py b/flux_local/tool/diff.py index 36ebbfa2..d5e525ce 100644 --- a/flux_local/tool/diff.py +++ b/flux_local/tool/diff.py @@ -179,7 +179,7 @@ def register( "-o", choices=["diff", "yaml"], default="diff", - help="If present, the namespace scope for this operation", + help="Output format of the command", ) args.set_defaults(cls=cls) return args diff --git a/flux_local/tool/flux_local.py b/flux_local/tool/flux_local.py index b319f6bd..24e719e9 100644 --- a/flux_local/tool/flux_local.py +++ b/flux_local/tool/flux_local.py @@ -8,7 +8,7 @@ import yaml -from . import build, diff, get, manifest, test +from . import build, diff, get, test from flux_local import command _LOGGER = logging.getLogger(__name__) @@ -51,14 +51,6 @@ def main() -> None: ) build_args.set_defaults(cls=build.BuildAction) - manifest_args = subparsers.add_parser( - "manifest", help="Build a yaml manifest file representing the cluster" - ) - manifest_args.add_argument( - "path", type=pathlib.Path, help="Path to the kustomization or charts" - ) - manifest_args.set_defaults(cls=manifest.ManifestAction) - test_args = subparsers.add_parser("test", help="Build and validate the cluster") test_args.add_argument( "path", type=pathlib.Path, help="Path to the kustomization or charts" diff --git a/flux_local/tool/format.py b/flux_local/tool/format.py index 17b8303a..f839438c 100644 --- a/flux_local/tool/format.py +++ b/flux_local/tool/format.py @@ -1,5 +1,8 @@ """Library for formatting output.""" +from typing import Generator, Any + +import yaml PADDING = 4 @@ -14,9 +17,52 @@ def column_format_string(rows: list[list[str]]) -> str: return "".join([f"{{:{w+PADDING}}}" for w in widths]) -def print_columns(headers: list[str], rows: list[list[str]]) -> None: +def format_columns( + headers: list[str], rows: list[list[str]] +) -> Generator[str, None, None]: """Print the specified output rows in a column format.""" data = [headers] + rows format_string = column_format_string(data) - for row in data: - print(format_string.format(*[str(x) for x in row])) + if format_string: + for row in data: + yield format_string.format(*[str(x) for x in row]) + + +class PrintFormatter: + """A formatter that prints human readable console output.""" + + def __init__(self, keys: list[str] | None = None): + """Initialize the PrintFormatter with optional keys to print.""" + self._keys = keys + + def format(self, data: list[dict[str, Any]]) -> Generator[str, None, None]: + """Format the data objects.""" + if not data: + return + keys = self._keys if self._keys is not None else list(data[0]) + rows = [] + for row in data: + rows.append([str(row[key]) for key in keys]) + cols = [col.upper() for col in keys] + for result in format_columns(cols, rows): + yield result + + def print(self, data: list[dict[str, Any]]) -> None: + """Output the data objects.""" + for result in self.format(data): + print(result) + + +class YamlFormatter: + """A formatter that prints yaml output.""" + + def format(self, data: list[dict[str, Any]]) -> Generator[str, None, None]: + """Format the data objects.""" + for line in yaml.dump_all(data, sort_keys=False, explicit_start=True).split( + "\n" + ): + yield line + + def print(self, data: list[dict[str, Any]]) -> None: + """Format the data objects.""" + print(yaml.dump_all(data, sort_keys=False, explicit_start=True)) diff --git a/flux_local/tool/get.py b/flux_local/tool/get.py index 22e21940..b77e608a 100644 --- a/flux_local/tool/get.py +++ b/flux_local/tool/get.py @@ -3,11 +3,11 @@ import logging from argparse import ArgumentParser from argparse import _SubParsersAction as SubParsersAction -from typing import cast +from typing import cast, Any from flux_local import git_repo -from .format import print_columns +from .format import PrintFormatter, YamlFormatter from . import selector @@ -44,26 +44,24 @@ async def run( # type: ignore[no-untyped-def] """Async Action implementation.""" query = selector.build_ks_selector(**kwargs) manifest = await git_repo.build_manifest(selector=query) - cols = ["NAME", "PATH", "HELMREPOS", "RELEASES"] + + results: list[dict[str, str]] = [] + cols = ["name", "path", "helmrepos", "releases"] if len(manifest.clusters) > 1: - cols.insert(0, "CLUSTER") - results: list[list[str]] = [] + cols.insert(0, "cluster") for cluster in manifest.clusters: for ks in cluster.kustomizations: - value = [ - ks.name, - ks.path, - str(len(ks.helm_repos)), - str(len(ks.helm_releases)), - ] - if len(manifest.clusters) > 1: - value.insert(0, cluster.path) + value = ks.dict(include=set(cols)) + value["helmrepos"] = len(ks.helm_repos) + value["releases"] = len(ks.helm_releases) + value["cluster"] = cluster.path results.append(value) + if not results: print(selector.not_found("Kustomization", query.kustomization)) return - print_columns(cols, results) + PrintFormatter(cols).print(results) class GetHelmReleaseAction: @@ -94,26 +92,81 @@ async def run( # type: ignore[no-untyped-def] """Async Action implementation.""" query = selector.build_hr_selector(**kwargs) manifest = await git_repo.build_manifest(selector=query) - cols = ["NAME", "REVISION", "CHART", "SOURCE"] + + cols = ["name", "revision", "chart", "source"] if query.helm_release.namespace is None: - cols.insert(0, "NAMESPACE") - results: list[list[str]] = [] + cols.insert(0, "namespace") + results: list[dict[str, Any]] = [] for cluster in manifest.clusters: for helmrelease in cluster.helm_releases: - value: list[str] = [ - helmrelease.name, - str(helmrelease.chart.version), - f"{helmrelease.namespace}-{helmrelease.chart.name}", - helmrelease.chart.repo_name, - ] - if query.helm_release.namespace is None: - value.insert(0, helmrelease.namespace) + value = helmrelease.dict(include=set(cols)) + value["revision"] = str(helmrelease.chart.version) + value["chart"] = f"{helmrelease.namespace}-{helmrelease.chart.name}" + value["source"] = helmrelease.chart.repo_name results.append(value) + if not results: print(selector.not_found("HelmRelease", query.helm_release)) return - print_columns(cols, results) + PrintFormatter(cols).print(results) + + +class GetClusterAction: + """Get details about flux clustaers.""" + + @classmethod + def register( + cls, subparsers: SubParsersAction # type: ignore[type-arg] + ) -> ArgumentParser: + """Register the subparser commands.""" + args = cast( + ArgumentParser, + subparsers.add_parser( + "clusters", + aliases=["cl", "cluster"], + help="Get get flux cluster definitions", + description="Print information about local flux cluster definitions", + ), + ) + selector.add_cluster_selector_flags(args) + args.add_argument( + "--output", + "-o", + choices=["diff", "yaml"], + default="diff", + help="Output format of the command", + ) + args.set_defaults(cls=cls) + return args + + async def run( # type: ignore[no-untyped-def] + self, + output: str, + **kwargs, # pylint: disable=unused-argument + ) -> None: + """Async Action implementation.""" + query = selector.build_cluster_selector(**kwargs) + query.helm_release.enabled = output == "yaml" + manifest = await git_repo.build_manifest(selector=query) + if output == "yaml": + YamlFormatter().print([manifest.compact_dict()]) + return + + cols = ["name", "path", "kustomizations"] + if query.cluster.namespace is None: + cols.insert(0, "namespace") + results: list[dict[str, Any]] = [] + for cluster in manifest.clusters: + value: dict[str, Any] = cluster.dict(include=set(cols)) + value["kustomizations"] = len(cluster.kustomizations) + results.append(value) + + if not results: + print(selector.not_found("flux cluster Kustmization", query.cluster)) + return + + PrintFormatter(cols).print(results) class GetAction: @@ -138,6 +191,7 @@ def register( ) GetKustomizationAction.register(subcmds) GetHelmReleaseAction.register(subcmds) + GetClusterAction.register(subcmds) args.set_defaults(cls=cls) return args diff --git a/flux_local/tool/manifest.py b/flux_local/tool/manifest.py deleted file mode 100644 index 7e038fca..00000000 --- a/flux_local/tool/manifest.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Flux-local manifest action.""" - -import pathlib - -from flux_local import git_repo - - -class ManifestAction: - """Flux-local manifest action.""" - - async def run( # type: ignore[no-untyped-def] - self, - path: pathlib.Path, - **kwargs, # pylint: disable=unused-argument - ) -> None: - """Async Action implementation.""" - manifest = await git_repo.build_manifest(path) - print(manifest.yaml()) diff --git a/flux_local/tool/selector.py b/flux_local/tool/selector.py index 0c262844..7997aa1f 100644 --- a/flux_local/tool/selector.py +++ b/flux_local/tool/selector.py @@ -88,6 +88,24 @@ def build_hr_selector( # type: ignore[no-untyped-def] return selector +def add_cluster_selector_flags(args: ArgumentParser) -> None: + """Add common flux cluster selector flags to the arguments object.""" + add_selector_flags(args) + + +def build_cluster_selector( # type: ignore[no-untyped-def] + **kwargs, +) -> git_repo.ResourceSelector: + """Build a selector object form the specified flags.""" + _LOGGER.debug("Building flux cluster Kustomization selector from args: %s", kwargs) + selector = git_repo.ResourceSelector() + selector.path = git_repo.PathSelector(kwargs.get("path")) + selector.cluster.namespace = kwargs.get("namespace") + if kwargs.get("all_namespaces"): + selector.cluster.namespace = None + return selector + + def not_found(resource: str, mds: git_repo.MetadataSelector) -> str: """Return a not found error message for the given resource type and query.""" if mds.name: diff --git a/tests/test_git_repo.py b/tests/test_git_repo.py index 32295934..d5763277 100644 --- a/tests/test_git_repo.py +++ b/tests/test_git_repo.py @@ -2,7 +2,7 @@ from pathlib import Path -from flux_local.git_repo import build_manifest +from flux_local.git_repo import build_manifest, ResourceSelector TESTDATA = Path("tests/testdata/cluster") @@ -12,5 +12,55 @@ async def test_build_manifest() -> None: manifest = await build_manifest(TESTDATA) assert len(manifest.clusters) == 1 - assert manifest.clusters[0].name == "flux-system" - assert manifest.clusters[0].path == "./tests/testdata/cluster/clusters/prod" + cluster = manifest.clusters[0] + assert cluster.name == "flux-system" + assert cluster.namespace == "flux-system" + assert cluster.path == "./tests/testdata/cluster/clusters/prod" + assert len(cluster.kustomizations) == 3 + assert len(cluster.helm_repos) == 2 + assert len(cluster.helm_releases) == 2 + + +async def test_cluster_selector_disabled() -> None: + """Tests for building the manifest.""" + + query = ResourceSelector() + query.path.path = TESTDATA + query.cluster.enabled = False + + manifest = await build_manifest(selector=query) + assert len(manifest.clusters) == 0 + + +async def test_kustomization_selector_disabled() -> None: + """Tests for building the manifest.""" + + query = ResourceSelector() + query.path.path = TESTDATA + query.kustomization.enabled = False + + manifest = await build_manifest(selector=query) + assert len(manifest.clusters) == 1 + cluster = manifest.clusters[0] + assert cluster.name == "flux-system" + assert cluster.namespace == "flux-system" + assert cluster.path == "./tests/testdata/cluster/clusters/prod" + assert len(cluster.kustomizations) == 0 + + +async def test_helm_release_selector_disabled() -> None: + """Tests for building the manifest.""" + + query = ResourceSelector() + query.path.path = TESTDATA + query.helm_release.enabled = False + + manifest = await build_manifest(selector=query) + assert len(manifest.clusters) == 1 + cluster = manifest.clusters[0] + assert cluster.name == "flux-system" + assert cluster.namespace == "flux-system" + assert cluster.path == "./tests/testdata/cluster/clusters/prod" + assert len(cluster.kustomizations) == 3 + assert len(cluster.helm_repos) == 0 + assert len(cluster.helm_releases) == 0 diff --git a/tests/test_helm.py b/tests/test_helm.py index 786e78b7..e5ba8b9e 100644 --- a/tests/test_helm.py +++ b/tests/test_helm.py @@ -36,7 +36,7 @@ async def helm_fixture(tmp_config_path: Path, helm_repos: list[dict[str, Any]]) tmp_config_path / "helm", tmp_config_path / "cache", ) - helm.add_repos([HelmRepository.from_doc(repo) for repo in helm_repos]) + helm.add_repos([HelmRepository.parse_doc(repo) for repo in helm_repos]) return helm @@ -58,7 +58,7 @@ async def test_template(helm: Helm, helm_releases: list[dict[str, Any]]) -> None assert len(helm_releases) == 1 release = helm_releases[0] - obj = await helm.template(HelmRelease.from_doc(release)) + obj = await helm.template(HelmRelease.parse_doc(release)) docs = await obj.grep("kind=ServiceAccount").objects() names = [doc.get("metadata", {}).get("name") for doc in docs] assert names == ["metallb-controller", "metallb-speaker"] diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 4e1fea24..6c14bb06 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -1,6 +1,5 @@ """Tests for manifest library.""" -import json from pathlib import Path import pytest @@ -22,7 +21,7 @@ def test_parse_helm_release() -> None: """Test parsing a helm release doc.""" - release = HelmRelease.from_doc( + release = HelmRelease.parse_doc( yaml.load( (TESTDATA_DIR / "controllers/metallb-release.yaml").read_text(), Loader=yaml.CLoader, @@ -34,6 +33,28 @@ def test_parse_helm_release() -> None: assert release.chart.version == "4.1.14" assert release.chart.repo_name == "bitnami" assert release.chart.repo_namespace == "flux-system" + assert release.values + + +def test_compact_helm_release() -> None: + """Test parsing a helm release doc.""" + + release = HelmRelease.parse_doc( + yaml.load( + (TESTDATA_DIR / "controllers/metallb-release.yaml").read_text(), + Loader=yaml.CLoader, + ) + ) + print(release) + assert release.compact_dict() == { + "name": "metallb", + "namespace": "metallb", + "chart": { + "name": "metallb", + "repo_name": "bitnami", + "repo_namespace": "flux-system", + }, + } def test_parse_helm_repository() -> None: @@ -43,18 +64,12 @@ def test_parse_helm_repository() -> None: (TESTDATA_DIR / "configs/helm-repositories.yaml").read_text(), Loader=yaml.CLoader, ) - repo = HelmRepository.from_doc(next(iter(docs))) + repo = HelmRepository.parse_doc(next(iter(docs))) assert repo.name == "bitnami" assert repo.namespace == "flux-system" assert repo.url == "https://charts.bitnami.com/bitnami" -async def test_read_manifest_invalid_file() -> None: - """Test reading an invalid manifest file.""" - with pytest.raises(ValidationError, match="validation error for Manifest"): - await read_manifest(Path("/dev/null")) - - async def test_write_manifest_file() -> None: """Test reading an invalid manifest file.""" await write_manifest(Path("/dev/null"), Manifest(clusters=[])) @@ -68,19 +83,35 @@ async def test_read_write_empty_manifest(tmp_path: Path) -> None: assert not new_manifest.clusters -async def test_read_write_manifest(tmp_path: Path) -> None: - """Test serializing and reading back a manifest.""" +async def test_read_manifest_invalid_file() -> None: + """Test reading an invalid manifest file.""" + with pytest.raises(ValidationError, match="validation error for Manifest"): + await read_manifest(Path("/dev/null")) + + +async def test_serializing_manifest(tmp_path: Path) -> None: + """Test serializing a manifest to a dictionary.""" manifest = Manifest( - clusters=[Cluster(name="cluster", path="./example", kustomizations=[])] + clusters=[ + Cluster( + name="cluster", + namespace="flux-system", + path="./example", + kustomizations=[], + ) + ] ) await write_manifest(tmp_path / "file.yaml", manifest) new_manifest = await read_manifest(tmp_path / "file.yaml") - assert json.loads(new_manifest.json()) == { + assert new_manifest.dict() == { "clusters": [ { "name": "cluster", + "namespace": "flux-system", "path": "./example", "kustomizations": [], }, ] } + assert new_manifest.compact_dict() == new_manifest.dict() + assert new_manifest.compact_dict() == manifest.dict() diff --git a/tests/test_flux_local.py b/tests/tool/test_flux_local.py similarity index 92% rename from tests/test_flux_local.py rename to tests/tool/test_flux_local.py index ba6376ba..48ba2261 100644 --- a/tests/test_flux_local.py +++ b/tests/tool/test_flux_local.py @@ -9,7 +9,7 @@ TESTDATA = "tests/testdata/cluster/" -@pytest.mark.golden_test("testdata/flux_local/*.yaml") +@pytest.mark.golden_test("testdata/*.yaml") async def test_flux_local_golden(golden: GoldenTestFixture) -> None: """Test commands in golden files.""" args = golden["args"] diff --git a/tests/tool/test_format.py b/tests/tool/test_format.py new file mode 100644 index 00000000..33efc134 --- /dev/null +++ b/tests/tool/test_format.py @@ -0,0 +1,109 @@ +"""Tests for the format library.""" + +from flux_local.tool.format import format_columns, PrintFormatter, YamlFormatter + + +def test_format_columns_empty() -> None: + """Tests with no rows.""" + assert list(format_columns([], [])) == [] + + +def test_format_columns_empty_rows() -> None: + """Tests with no rows.""" + assert list(format_columns(["a", "b", "c"], [])) == ["a b c "] + + +def test_format_columns_rws() -> None: + """Tests format with normal rows""" + assert list( + format_columns( + ["name", "namespace"], [["podinfo", "podinfo"], ["metallb", "network"]] + ) + ) == [ + "name namespace ", + "podinfo podinfo ", + "metallb network ", + ] + + +def test_print_formatter() -> None: + """Print formatting with empty data.""" + formatter = PrintFormatter() + assert list(formatter.format([])) == [] + + +def test_print_formatter_data() -> None: + """Print formatting data objects.""" + formatter = PrintFormatter() + assert list( + formatter.format( + [ + { + "name": "podinfo", + "namespace": "podinfo", + }, + { + "name": "metallb", + "namespace": "network", + }, + ] + ) + ) == [ + "NAME NAMESPACE ", + "podinfo podinfo ", + "metallb network ", + ] + + +def test_print_formatter_keys() -> None: + """Print formatting with column names.""" + formatter = PrintFormatter(keys=["name"]) + assert list( + formatter.format( + [ + { + "name": "podinfo", + "namespace": "podinfo", + }, + { + "name": "metallb", + "namespace": "network", + }, + ], + ) + ) == [ + "NAME ", + "podinfo ", + "metallb ", + ] + + +def test_yaml_formatter() -> None: + """Print formatting with column names.""" + formatter = YamlFormatter() + assert list( + formatter.format( + [ + { + "kustomizations": [ + { + "name": "podinfo", + "namespace": "podinfo", + }, + { + "name": "metallb", + "namespace": "network", + }, + ], + } + ] + ) + ) == [ + "---", + "kustomizations:", + "- name: podinfo", + " namespace: podinfo", + "- name: metallb", + " namespace: network", + "", + ] diff --git a/tests/testdata/flux_local/build.yaml b/tests/tool/testdata/build.yaml similarity index 100% rename from tests/testdata/flux_local/build.yaml rename to tests/tool/testdata/build.yaml diff --git a/tests/testdata/flux_local/build_helm.yaml b/tests/tool/testdata/build_helm.yaml similarity index 100% rename from tests/testdata/flux_local/build_helm.yaml rename to tests/tool/testdata/build_helm.yaml diff --git a/tests/testdata/flux_local/diff_hr.yaml b/tests/tool/testdata/diff_hr.yaml similarity index 100% rename from tests/testdata/flux_local/diff_hr.yaml rename to tests/tool/testdata/diff_hr.yaml diff --git a/tests/testdata/flux_local/diff_hr_all.yaml b/tests/tool/testdata/diff_hr_all.yaml similarity index 100% rename from tests/testdata/flux_local/diff_hr_all.yaml rename to tests/tool/testdata/diff_hr_all.yaml diff --git a/tests/testdata/flux_local/diff_hr_all_namespaces.yaml b/tests/tool/testdata/diff_hr_all_namespaces.yaml similarity index 100% rename from tests/testdata/flux_local/diff_hr_all_namespaces.yaml rename to tests/tool/testdata/diff_hr_all_namespaces.yaml diff --git a/tests/testdata/flux_local/diff_hr_not_found.yaml b/tests/tool/testdata/diff_hr_not_found.yaml similarity index 100% rename from tests/testdata/flux_local/diff_hr_not_found.yaml rename to tests/tool/testdata/diff_hr_not_found.yaml diff --git a/tests/testdata/flux_local/diff_hr_not_found_all_namespaces.yaml b/tests/tool/testdata/diff_hr_not_found_all_namespaces.yaml similarity index 100% rename from tests/testdata/flux_local/diff_hr_not_found_all_namespaces.yaml rename to tests/tool/testdata/diff_hr_not_found_all_namespaces.yaml diff --git a/tests/testdata/flux_local/diff_hr_not_found_namespace.yaml b/tests/tool/testdata/diff_hr_not_found_namespace.yaml similarity index 100% rename from tests/testdata/flux_local/diff_hr_not_found_namespace.yaml rename to tests/tool/testdata/diff_hr_not_found_namespace.yaml diff --git a/tests/testdata/flux_local/diff_ks.yaml b/tests/tool/testdata/diff_ks.yaml similarity index 100% rename from tests/testdata/flux_local/diff_ks.yaml rename to tests/tool/testdata/diff_ks.yaml diff --git a/tests/tool/testdata/get_cluster.yaml b/tests/tool/testdata/get_cluster.yaml new file mode 100644 index 00000000..ef663a01 --- /dev/null +++ b/tests/tool/testdata/get_cluster.yaml @@ -0,0 +1,8 @@ +args: +- get +- cluster +- --path +- tests/testdata/cluster/ +stdout: | + NAME PATH KUSTOMIZATIONS + flux-system ./tests/testdata/cluster/clusters/prod 3 diff --git a/tests/tool/testdata/get_cluster_all.yaml b/tests/tool/testdata/get_cluster_all.yaml new file mode 100644 index 00000000..ddd50163 --- /dev/null +++ b/tests/tool/testdata/get_cluster_all.yaml @@ -0,0 +1,9 @@ +args: +- get +- cluster +- --all-namespaces +- --path +- tests/testdata/cluster/ +stdout: | + NAMESPACE NAME PATH KUSTOMIZATIONS + flux-system flux-system ./tests/testdata/cluster/clusters/prod 3 diff --git a/tests/testdata/flux_local/manifest.yaml b/tests/tool/testdata/get_cluster_yaml.yaml similarity index 94% rename from tests/testdata/flux_local/manifest.yaml rename to tests/tool/testdata/get_cluster_yaml.yaml index 4be1e302..7e007ede 100644 --- a/tests/testdata/flux_local/manifest.yaml +++ b/tests/tool/testdata/get_cluster_yaml.yaml @@ -1,10 +1,15 @@ args: -- manifest +- get +- cluster +- --path - tests/testdata/cluster/ +- -o +- yaml stdout: |+ --- clusters: - name: flux-system + namespace: flux-system path: ./tests/testdata/cluster/clusters/prod kustomizations: - name: apps diff --git a/tests/testdata/flux_local/get_hr.yaml b/tests/tool/testdata/get_hr.yaml similarity index 100% rename from tests/testdata/flux_local/get_hr.yaml rename to tests/tool/testdata/get_hr.yaml diff --git a/tests/testdata/flux_local/get_hr_name.yaml b/tests/tool/testdata/get_hr_name.yaml similarity index 100% rename from tests/testdata/flux_local/get_hr_name.yaml rename to tests/tool/testdata/get_hr_name.yaml diff --git a/tests/testdata/flux_local/get_hr_namespace.yaml b/tests/tool/testdata/get_hr_namespace.yaml similarity index 100% rename from tests/testdata/flux_local/get_hr_namespace.yaml rename to tests/tool/testdata/get_hr_namespace.yaml diff --git a/tests/testdata/flux_local/get_ks.yaml b/tests/tool/testdata/get_ks.yaml similarity index 100% rename from tests/testdata/flux_local/get_ks.yaml rename to tests/tool/testdata/get_ks.yaml