From 685730a6ba3753427105d8f9045f59147e879d86 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 6 Feb 2023 00:22:10 +0000 Subject: [PATCH 1/4] Add commands for evaluating HelmReleases --- flux_local/helm.py | 98 ++++++++++++++++++- flux_local/kustomize.py | 63 ++++++++---- flux_local/manifest.py | 66 +++++++++---- tests/test_helm.py | 71 ++++++++++++++ tests/test_kustomize.py | 24 +++-- tests/test_manifest.py | 55 ++--------- tests/testdata/helm-repo/kustomization.yaml | 6 ++ tests/testdata/helm-repo/metallb-release.yaml | 24 +++++ tests/testdata/helm-repo/sources.yaml | 11 +++ .../testdata/{repo.golden => repo/all.golden} | 0 .../configmap.golden} | 0 11 files changed, 320 insertions(+), 98 deletions(-) create mode 100644 tests/test_helm.py create mode 100644 tests/testdata/helm-repo/kustomization.yaml create mode 100644 tests/testdata/helm-repo/metallb-release.yaml create mode 100644 tests/testdata/helm-repo/sources.yaml rename tests/testdata/{repo.golden => repo/all.golden} (100%) rename tests/testdata/{repo.configmap.golden => repo/configmap.golden} (100%) diff --git a/flux_local/helm.py b/flux_local/helm.py index db977511..809ffdad 100644 --- a/flux_local/helm.py +++ b/flux_local/helm.py @@ -1,9 +1,97 @@ """Library for running `helm template` to produce local items in the cluster.""" -import subprocess +import datetime +import logging +from pathlib import Path +from typing import Any, Generator +import aiofiles +import yaml -def render(chart, values): - """Runs `helm template` to produce local items in the cluster.""" - cmd = ["helm", "template", chart, "--values", values] - return subprocess.check_output(cmd) +from .manifest import HelmRepository, HelmRelease +from .kustomize import Kustomize +from . import command + + +__all__ = [ + "Helm", +] + +_LOGGER = logging.getLogger(__name__) + + +HELM_BIN = "helm" + + +class RepositoryConfig: + """Generates a helm repository configuration from flux HelmRepository objects.""" + + def __init__(self, repos: list[HelmRepository]) -> None: + """Initialize RepositoryConfig.""" + self._repos = repos + + @property + def config(self) -> dict[str, Any]: + """Return a synthetic repostory config object.""" + now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) + return { + "apiVersion": "", + "generated": now.isoformat(), + "repositories": [ + { + "name": f"{repo.namespace}-{repo.name}", + "url": repo.url, + } for repo in self._repos + ], + } + + +class Helm: + """Manages local HelmRepository state.""" + + def __init__(self, tmp_dir: Path, cache_dir: Path) -> None: + """Initialize Helm.""" + self._tmp_dir = tmp_dir + self._repo_config_file = self._tmp_dir / "repository-config.yaml" + self._flags = [ + "--registry-config", "/dev/null", + "--repository-cache", str(cache_dir), + "--repository-config", str(self._repo_config_file), + ] + self._repos: list[HelmRepository] = [] + + def add_repo(self, repo: HelmRepository): + """Add the specified HelmRepository to the local config.""" + self._repos.append(repo) + + async def update(self) -> None: + """Return command line arguments to update the local repo. + + Typically the repository must be updated before doing any chart templating. + """ + content = yaml.dump(RepositoryConfig(self._repos).config) + async with aiofiles.open(str(self._repo_config_file), mode="w") as f: + await f.write(content) + await command.run([ HELM_BIN, "repo", "update" ] + self._flags) + + async def template( + self, + release: HelmRelease, + values: dict[str, Any] + ) -> Kustomize: + """Return command line arguments to template the specified chart.""" + args = [ + HELM_BIN, + "template", + release.name, + release.chart.chart_name, + "--namespace", release.namespace, + "--skip-crds", # Reduce size of output + "--version", release.chart.version, + ] + if values: + values_file = tmp_config_path / f"{release.release_name}-values.yaml" + async with aiofiles.open(values_file, mode="w") as f: + await f.write(yaml.dump(values)) + args.extend(["--values", str(values_file)]) + return Kustomize([args + self._flags]) diff --git a/flux_local/kustomize.py b/flux_local/kustomize.py index 666db774..92f0e912 100644 --- a/flux_local/kustomize.py +++ b/flux_local/kustomize.py @@ -6,27 +6,50 @@ """ from pathlib import Path +from typing import Generator, Any +import yaml -KUSTOMIZE_BIN = "kustomize" - - -def build(path: Path) -> list[str]: - """Generate a kustomize build command for the specified path.""" - return [KUSTOMIZE_BIN, "build", str(path)] +from . import command +KUSTOMIZE_BIN = "kustomize" -def grep(expr: str, path: Path | None = None, invert: bool = False) -> list[str]: - """Generate a kustomize grep command to filter resources based on an expression. - - Example expressions: - `kind=HelmRelease` - `metadata.name=redis` - The return value is a set of command args. - """ - out = [KUSTOMIZE_BIN, "cfg", "grep", expr] - if invert: - out.append("--invert-match") - if path: - out.append(str(path)) - return out +class Kustomize: + """Library for issuing a kustomize command.""" + + def __init__(self, cmds: list[list[str]]) -> None: + """Initialize Kustomize.""" + self._cmds = cmds + + @classmethod + def build(cls, path: Path) -> 'Kustomize': + """Build cluster artifacts from the specified path.""" + return Kustomize(cmds=[[KUSTOMIZE_BIN, "build", str(path)]]) + + def grep(self, expr: str, path: Path | None = None, invert: bool = False) -> 'Kustomize': + """Filter resources based on an expression. + + Example expressions: + `kind=HelmRelease` + `metadata.name=redis` + """ + out = [KUSTOMIZE_BIN, "cfg", "grep", expr] + if invert: + out.append("--invert-match") + if path: + out.append(str(path)) + return Kustomize(self._cmds + [out]) + + async def run(self) -> str: + """Run the kustomize command and return the output as a string.""" + return await command.run_piped(self._cmds) + + async def _docs(self) -> Generator[dict[str, Any], None, None]: + """Run the kustomize command and return the result documents.""" + out = await self.run() + for doc in yaml.safe_load_all(out): + yield doc + + async def docs(self) -> list[dict[str, Any]]: + """Run the kustomize command and return the result documents as a list.""" + return [ doc async for doc in self._docs() ] diff --git a/flux_local/manifest.py b/flux_local/manifest.py index 953dc8ec..eb400a97 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -17,18 +17,19 @@ "Kustomization", "HelmRepository", "HelmRelease", + "HelmChart", ] @dataclass -class HelmRelease: - """A representation of a Flux HelmRelease.""" +class HelmChart: + """A representation of an instantiation of a chart for a HelmRelease.""" name: str - """The name of the HelmRelease.""" + """The name of the chart within the HelmRepository.""" - namespace: str - """The namespace that owns the HelmRelease.""" + version: str + """The version of the chart.""" repo_name: str """The name of the HelmRepository.""" @@ -37,29 +38,58 @@ class HelmRelease: """The namespace of the HelmRepository.""" @classmethod - def from_doc(cls, doc: dict[str, Any]) -> "HelmRelease": - """Parse a HelmRelease from a kubernetes resource object.""" - if not (metadata := doc.get("metadata")): - raise ValueError(f"Invalid {cls} missing metadata: {doc}") - if not (name := metadata.get("name")): - raise ValueError(f"Invalid {cls} missing metadata.name: {doc}") - if not (namespace := metadata.get("namespace")): - raise ValueError(f"Invalid {cls} missing metadata.namespace: {doc}") + def from_doc(cls, doc: dict[str, Any]) -> "HelmChart": + """Parse a HelmChart from a HelmRelease resource object.""" if not (spec := doc.get("spec")): raise ValueError(f"Invalid {cls} missing spec: {doc}") if not (chart := spec.get("chart")): raise ValueError(f"Invalid {cls} missing spec.chart: {doc}") if not (chart_spec := chart.get("spec")): raise ValueError(f"Invalid {cls} missing spec.chart.spec: {doc}") + if not (chart := chart_spec.get("chart")): + raise ValueError(f"Invalid {cls} missing spec.chart.spec.chart: {doc}") + if not (version := chart_spec.get("version")): + raise ValueError(f"Invalid {cls} missing spec.chart.spec.version: {doc}") if not (source_ref := chart_spec.get("sourceRef")): raise ValueError(f"Invalid {cls} missing spec.chart.spec.sourceRef: {doc}") if "namespace" not in source_ref or "name" not in source_ref: raise ValueError(f"Invalid {cls} missing sourceRef fields: {doc}") - return cls(name, namespace, source_ref["name"], source_ref["namespace"]) + return cls(chart, version, source_ref["name"], source_ref["namespace"]) @property - def id_name(self) -> str: - """Identifier for the HelmRelease in tests.""" + def chart_name(self) -> str: + """Identifer for the HelmChart.""" + return f"{self.repo_namespace}-{self.repo_name}/{self.name}" + + +@dataclass +class HelmRelease: + """A representation of a Flux HelmRelease.""" + + name: str + """The name of the HelmRelease.""" + + namespace: str + """The namespace that owns the HelmRelease.""" + + chart: HelmChart + """A mapping to a specific helm chart for this HelmRelease.""" + + @classmethod + def from_doc(cls, doc: dict[str, Any]) -> "HelmRelease": + """Parse a HelmRelease from a kubernetes resource object.""" + if not (metadata := doc.get("metadata")): + raise ValueError(f"Invalid {cls} missing metadata: {doc}") + if not (name := metadata.get("name")): + raise ValueError(f"Invalid {cls} missing metadata.name: {doc}") + if not (namespace := metadata.get("namespace")): + raise ValueError(f"Invalid {cls} missing metadata.namespace: {doc}") + chart = HelmChart.from_doc(doc) + return cls(name, namespace, chart) + + @property + def release_name(self) -> str: + """Identifier for the HelmRelease.""" return f"{self.namespace}-{self.name}" @@ -92,8 +122,8 @@ def from_doc(cls, doc: dict[str, Any]) -> "HelmRepository": return cls(name, namespace, url) @property - def id_name(self) -> str: - """Identifier for the HelmRepository in tests.""" + def repo_name(self) -> str: + """Identifier for the HelmRepository.""" return f"{self.namespace}-{self.name}" diff --git a/tests/test_helm.py b/tests/test_helm.py new file mode 100644 index 00000000..4592033a --- /dev/null +++ b/tests/test_helm.py @@ -0,0 +1,71 @@ +"""Tests for cmd library.""" + +from aiofiles.os import mkdir +from pathlib import Path +from typing import Any, Generator +import yaml + +import pytest + +from flux_local.manifest import HelmRepository +from flux_local.manifest import HelmRelease +from flux_local.kustomize import Kustomize +from flux_local.helm import Helm + +TESTDATA_DIR = Path("tests/testdata/") / "helm-repo" + +@pytest.fixture(name="tmp_config_path") +def tmp_config_path_fixture(tmp_path_factory: Any) -> Generator[Path, None, None]: + """Fixture for creating a path used for helm config shared across tests.""" + yield tmp_path_factory.mktemp("test_helm") + + +@pytest.fixture(name="helm_repos") +async def helm_repos_fixture() -> list[HelmRepository]: + """Fixture for creating the HelmRepository objects""" + kustomize = Kustomize.build(TESTDATA_DIR).grep("kind=^HelmRepository$") + return await kustomize.docs() + + +@pytest.fixture(name="helm") +async def helm_fixture(tmp_config_path: Path, helm_repos: list[dict[str, any]]) -> Helm: + """Fixture for creating the Helm object.""" + await mkdir(tmp_config_path / "helm") + await mkdir(tmp_config_path / "cache") + helm = Helm( + tmp_config_path / "helm", + tmp_config_path / "cache", + ) + for repo in helm_repos: + helm.add_repo(HelmRepository.from_doc(repo)) + return helm + + +@pytest.fixture(name="helm_releases") +async def helm_releases_fixture() -> list[dict[str, Any]]: + """Fixture for creating the HelmRelease objects.""" + kustomize = Kustomize.build(TESTDATA_DIR).grep("kind=^HelmRelease$") + return await kustomize.docs() + + +async def test_update(helm: Helm) -> None: + """Test a helm update command.""" + await helm.update() + + +async def test_template(helm: Helm, helm_releases: list[dict[str, Any]]) -> None: + """Test helm template command.""" + await helm.update() + + assert len(helm_releases) == 1 + release = helm_releases[0] + kustomize = await helm.template( + HelmRelease.from_doc(release), + release["spec"].get("values") + ) + docs = await kustomize.grep("kind=ServiceAccount").docs() + names = [ doc.get("metadata", {}).get("name") for doc in docs ] + assert names == [ + 'metallb-controller', + 'metallb-speaker' + ] diff --git a/tests/test_kustomize.py b/tests/test_kustomize.py index e53e77b5..7b9c3b44 100644 --- a/tests/test_kustomize.py +++ b/tests/test_kustomize.py @@ -2,27 +2,31 @@ from pathlib import Path -from flux_local import command, kustomize +from flux_local.kustomize import Kustomize TESTDATA_DIR = Path("tests/testdata") async def test_build() -> None: """Test a kustomize build command.""" - result = await command.run(kustomize.build(TESTDATA_DIR / "repo")) + result = await Kustomize.build(TESTDATA_DIR / "repo").run() assert "Secret" in result assert "ConfigMap" in result - assert result == (TESTDATA_DIR / "repo.golden").read_text() + assert result == (TESTDATA_DIR / "repo/all.golden").read_text() async def test_grep() -> None: """Test a kustomize build command.""" - result = await command.run_piped( - [ - kustomize.build(TESTDATA_DIR / "repo"), - kustomize.grep("kind=ConfigMap"), - ] - ) + result = await Kustomize.build(TESTDATA_DIR / "repo").grep("kind=ConfigMap").run() assert "Secret" not in result assert "ConfigMap" in result - assert result == (TESTDATA_DIR / "repo.configmap.golden").read_text() + assert result == (TESTDATA_DIR / "repo/configmap.golden").read_text() + + +async def test_docs() -> None: + """Test loading yaml documents.""" + kustomize = Kustomize.build(TESTDATA_DIR / "repo").grep("kind=ConfigMap") + result = await kustomize.docs() + assert len(result) == 1 + assert result[0].get("kind") == "ConfigMap" + assert result[0].get("apiVersion") == "v1" diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 66c45428..eea92e0c 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -1,68 +1,33 @@ """Tests for manifest library.""" import yaml +from pathlib import Path from flux_local.manifest import HelmRelease, HelmRepository +TESTDATA_DIR = Path("tests/testdata/helm-repo") + + def test_parse_helm_release() -> None: """Test parsing a helm release doc.""" release = HelmRelease.from_doc( - yaml.load( - """--- -apiVersion: helm.toolkit.fluxcd.io/v2beta1 -kind: HelmRelease -metadata: - name: metallb - namespace: metallb -spec: - chart: - spec: - chart: metallb - reconcileStrategy: ChartVersion - sourceRef: - kind: HelmRepository - name: bitnami - namespace: flux-system - version: 4.1.14 - install: - crds: CreateReplace - remediation: - retries: 3 - interval: 5m - releaseName: metallb - upgrade: - crds: CreateReplace -""", - Loader=yaml.CLoader, - ) + yaml.load((TESTDATA_DIR / "metallb-release.yaml").read_text(), Loader=yaml.CLoader) ) assert release.name == "metallb" assert release.namespace == "metallb" - assert release.repo_name == "bitnami" - assert release.repo_namespace == "flux-system" + assert release.chart.name == "metallb" + assert release.chart.version == "4.1.14" + assert release.chart.repo_name == "bitnami" + assert release.chart.repo_namespace == "flux-system" def test_parse_helm_repository() -> None: """Test parsing a helm repository doc.""" repo = HelmRepository.from_doc( - yaml.load( - """--- -apiVersion: source.toolkit.fluxcd.io/v1beta2 -kind: HelmRepository -metadata: - name: bitnami - namespace: flux-system -spec: - interval: 30m - provider: generic - timeout: 1m0s - url: https://charts.bitnami.com/bitnami -""", - Loader=yaml.CLoader, - ) + yaml.load((TESTDATA_DIR / "sources.yaml").read_text(), Loader=yaml.CLoader) ) assert repo.name == "bitnami" assert repo.namespace == "flux-system" diff --git a/tests/testdata/helm-repo/kustomization.yaml b/tests/testdata/helm-repo/kustomization.yaml new file mode 100644 index 00000000..0e5f29aa --- /dev/null +++ b/tests/testdata/helm-repo/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - metallb-release.yaml + - sources.yaml diff --git a/tests/testdata/helm-repo/metallb-release.yaml b/tests/testdata/helm-repo/metallb-release.yaml new file mode 100644 index 00000000..3cd7b14c --- /dev/null +++ b/tests/testdata/helm-repo/metallb-release.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: metallb + namespace: metallb +spec: + chart: + spec: + chart: metallb + reconcileStrategy: ChartVersion + sourceRef: + kind: HelmRepository + name: bitnami + namespace: flux-system + version: 4.1.14 + install: + crds: CreateReplace + remediation: + retries: 3 + interval: 5m + releaseName: metallb + upgrade: + crds: CreateReplace diff --git a/tests/testdata/helm-repo/sources.yaml b/tests/testdata/helm-repo/sources.yaml new file mode 100644 index 00000000..3320a100 --- /dev/null +++ b/tests/testdata/helm-repo/sources.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: bitnami + namespace: flux-system +spec: + interval: 30m + provider: generic + timeout: 1m0s + url: https://charts.bitnami.com/bitnami diff --git a/tests/testdata/repo.golden b/tests/testdata/repo/all.golden similarity index 100% rename from tests/testdata/repo.golden rename to tests/testdata/repo/all.golden diff --git a/tests/testdata/repo.configmap.golden b/tests/testdata/repo/configmap.golden similarity index 100% rename from tests/testdata/repo.configmap.golden rename to tests/testdata/repo/configmap.golden From b9bcec4563df90548c2b5103d677af287bc33324 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 6 Feb 2023 00:47:36 +0000 Subject: [PATCH 2/4] Improve documentation for library commands --- flux_local/helm.py | 36 +++++++++++++++++++++++++++++++++++- flux_local/kustomize.py | 23 +++++++++++++++++++++-- tests/test_command.py | 2 +- tests/test_helm.py | 8 ++++---- tests/test_kustomize.py | 6 +++--- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/flux_local/helm.py b/flux_local/helm.py index 809ffdad..100924bd 100644 --- a/flux_local/helm.py +++ b/flux_local/helm.py @@ -1,4 +1,38 @@ -"""Library for running `helm template` to produce local items in the cluster.""" +"""Library for running `helm template` to produce local items in the cluster. + +You can instantiate a helm template with the following: +- A HelmRepository which is a url that contains charts +- A HelmRelease which is an instance of a HelmChart in a HelmRepository + +This is an example that prepares the helm repository: +``` +from flux_local.kustomize import Kustomize +from flux_local.helm import Helm +from flux_local.manifest import HelmRepository + +kustomize = Kustomize.build(TESTDATA_DIR) +repos = await kustomize.grep("kind=^HelmRepository$").objects() +helm = Helm("/tmp/path/helm", "/tmp/path/cache") +for repo in repos: + helm.add_repo(HelmRepository.from_doc(repo)) +await helm.update() +``` + +Then to actually instantiate a template from a HelmRelease: +``` +from flux_local.manifest import HelmRelease + +releases = await kustomize.grep("kind=^HelmRelease$").objects() +if not len(releases) == 1: + raise ValueError("Expected only one HelmRelease") +tmpl = helm.template( + HelmRelease.from_doc(releases[0]), + releases[0]["spec"].get("values")) +objects = await tmpl.objects() +for object in objects: + print(f"Found object {object['apiVersion']} {object['kind']}") +``` +""" import datetime import logging diff --git a/flux_local/kustomize.py b/flux_local/kustomize.py index 92f0e912..c63c4490 100644 --- a/flux_local/kustomize.py +++ b/flux_local/kustomize.py @@ -3,6 +3,25 @@ Kustomize build can be used to apply overlays and output a set of resources or artifacts for the cluster that can be either be parsed directly or piped into additional commands for processing and filtering by kustomize grep. + +This example returns the objects inside a Kustomization using `kustomize build`: +``` +from flux_local.kustomize import Kustomize + +objects = await Kustomize.build('/path/to/objects').objects() +for object in objects: + print(f"Found object {object['apiVersion']} {object['kind']}") +``` + +You can also filter documents to specific resource types or other fields: +``` +from flux_local.kustomize import Kustomize + +objects = await Kustomize.build('/path/to/objects').grep('kind=ConfigMap').objects() +for object in objects: + print(f"Found ConfigMap: {object['metadata']['name']}") +``` + """ from pathlib import Path @@ -50,6 +69,6 @@ async def _docs(self) -> Generator[dict[str, Any], None, None]: for doc in yaml.safe_load_all(out): yield doc - async def docs(self) -> list[dict[str, Any]]: - """Run the kustomize command and return the result documents as a list.""" + async def objects(self) -> list[dict[str, Any]]: + """Run the kustomize command and return the result cluster objects as a list.""" return [ doc async for doc in self._docs() ] diff --git a/tests/test_command.py b/tests/test_command.py index ff4db8f4..1aa8e972 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -1,4 +1,4 @@ -"""Tests for cmd library.""" +"""Tests for command library.""" import pytest diff --git a/tests/test_helm.py b/tests/test_helm.py index 4592033a..80821c4f 100644 --- a/tests/test_helm.py +++ b/tests/test_helm.py @@ -1,4 +1,4 @@ -"""Tests for cmd library.""" +"""Tests for helm library.""" from aiofiles.os import mkdir from pathlib import Path @@ -24,7 +24,7 @@ def tmp_config_path_fixture(tmp_path_factory: Any) -> Generator[Path, None, None async def helm_repos_fixture() -> list[HelmRepository]: """Fixture for creating the HelmRepository objects""" kustomize = Kustomize.build(TESTDATA_DIR).grep("kind=^HelmRepository$") - return await kustomize.docs() + return await kustomize.objects() @pytest.fixture(name="helm") @@ -45,7 +45,7 @@ async def helm_fixture(tmp_config_path: Path, helm_repos: list[dict[str, any]]) async def helm_releases_fixture() -> list[dict[str, Any]]: """Fixture for creating the HelmRelease objects.""" kustomize = Kustomize.build(TESTDATA_DIR).grep("kind=^HelmRelease$") - return await kustomize.docs() + return await kustomize.objects() async def test_update(helm: Helm) -> None: @@ -63,7 +63,7 @@ async def test_template(helm: Helm, helm_releases: list[dict[str, Any]]) -> None HelmRelease.from_doc(release), release["spec"].get("values") ) - docs = await kustomize.grep("kind=ServiceAccount").docs() + docs = await kustomize.grep("kind=ServiceAccount").objects() names = [ doc.get("metadata", {}).get("name") for doc in docs ] assert names == [ 'metallb-controller', diff --git a/tests/test_kustomize.py b/tests/test_kustomize.py index 7b9c3b44..52f59813 100644 --- a/tests/test_kustomize.py +++ b/tests/test_kustomize.py @@ -1,4 +1,4 @@ -"""Tests for cmd library.""" +"""Tests for kustomize library.""" from pathlib import Path @@ -23,10 +23,10 @@ async def test_grep() -> None: assert result == (TESTDATA_DIR / "repo/configmap.golden").read_text() -async def test_docs() -> None: +async def test_objects() -> None: """Test loading yaml documents.""" kustomize = Kustomize.build(TESTDATA_DIR / "repo").grep("kind=ConfigMap") - result = await kustomize.docs() + result = await kustomize.objects() assert len(result) == 1 assert result[0].get("kind") == "ConfigMap" assert result[0].get("apiVersion") == "v1" From 7209266cef750fff2343986163d5998a1e24b471 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 6 Feb 2023 01:05:54 +0000 Subject: [PATCH 3/4] Address lint and style issues --- .pre-commit-config.yaml | 19 ++++++++------- flux_local/helm.py | 51 +++++++++++++++++++++-------------------- flux_local/kustomize.py | 15 +++++++----- requirements.txt | 1 + script/run-in-env.sh | 22 ++++++++++++++++++ setup.cfg | 5 ++-- 6 files changed, 70 insertions(+), 43 deletions(-) create mode 100755 script/run-in-env.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7bb1119c..3cb57896 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - -c - ".yaml-lint.yaml" - repo: https://github.com/pycqa/isort - rev: 5.9.3 + rev: 5.12.0 hooks: - id: isort args: ["--profile", "black"] @@ -27,20 +27,19 @@ repos: rev: 22.3.0 hooks: - id: black -- repo: https://gitlab.com/pycqa/flake8 - rev: 4.0.1 +- repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 hooks: - id: flake8 -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 +- repo: local hooks: - id: mypy + name: mypy + entry: script/run-in-env.sh mypy + language: script + types: [python] + require_serial: true files: ^(flux_local/|tests/) - additional_dependencies: - - "pydantic>=1.10.4" - - "types-PyYAML>=6.0.12.4" -- repo: local - hooks: - id: pylint name: pylint entry: pylint diff --git a/flux_local/helm.py b/flux_local/helm.py index 100924bd..9e96684c 100644 --- a/flux_local/helm.py +++ b/flux_local/helm.py @@ -37,18 +37,17 @@ import datetime import logging from pathlib import Path -from typing import Any, Generator +from typing import Any import aiofiles import yaml -from .manifest import HelmRepository, HelmRelease -from .kustomize import Kustomize from . import command - +from .kustomize import Kustomize +from .manifest import HelmRelease, HelmRepository __all__ = [ - "Helm", + "Helm", ] _LOGGER = logging.getLogger(__name__) @@ -75,7 +74,8 @@ def config(self) -> dict[str, Any]: { "name": f"{repo.namespace}-{repo.name}", "url": repo.url, - } for repo in self._repos + } + for repo in self._repos ], } @@ -88,13 +88,16 @@ def __init__(self, tmp_dir: Path, cache_dir: Path) -> None: self._tmp_dir = tmp_dir self._repo_config_file = self._tmp_dir / "repository-config.yaml" self._flags = [ - "--registry-config", "/dev/null", - "--repository-cache", str(cache_dir), - "--repository-config", str(self._repo_config_file), + "--registry-config", + "/dev/null", + "--repository-cache", + str(cache_dir), + "--repository-config", + str(self._repo_config_file), ] self._repos: list[HelmRepository] = [] - def add_repo(self, repo: HelmRepository): + def add_repo(self, repo: HelmRepository) -> None: """Add the specified HelmRepository to the local config.""" self._repos.append(repo) @@ -104,28 +107,26 @@ async def update(self) -> None: Typically the repository must be updated before doing any chart templating. """ content = yaml.dump(RepositoryConfig(self._repos).config) - async with aiofiles.open(str(self._repo_config_file), mode="w") as f: - await f.write(content) - await command.run([ HELM_BIN, "repo", "update" ] + self._flags) - - async def template( - self, - release: HelmRelease, - values: dict[str, Any] - ) -> Kustomize: + async with aiofiles.open(str(self._repo_config_file), mode="w") as config_file: + await config_file.write(content) + await command.run([HELM_BIN, "repo", "update"] + self._flags) + + async def template(self, release: HelmRelease, values: dict[str, Any]) -> Kustomize: """Return command line arguments to template the specified chart.""" args = [ HELM_BIN, "template", release.name, release.chart.chart_name, - "--namespace", release.namespace, + "--namespace", + release.namespace, "--skip-crds", # Reduce size of output - "--version", release.chart.version, + "--version", + release.chart.version, ] if values: - values_file = tmp_config_path / f"{release.release_name}-values.yaml" - async with aiofiles.open(values_file, mode="w") as f: - await f.write(yaml.dump(values)) - args.extend(["--values", str(values_file)]) + values_path = self._tmp_dir / f"{release.release_name}-values.yaml" + async with aiofiles.open(values_path, mode="w") as values_file: + await values_file.write(yaml.dump(values)) + args.extend(["--values", str(values_path)]) return Kustomize([args + self._flags]) diff --git a/flux_local/kustomize.py b/flux_local/kustomize.py index c63c4490..b8488098 100644 --- a/flux_local/kustomize.py +++ b/flux_local/kustomize.py @@ -25,7 +25,8 @@ """ from pathlib import Path -from typing import Generator, Any +from typing import Any, AsyncGenerator + import yaml from . import command @@ -41,11 +42,13 @@ def __init__(self, cmds: list[list[str]]) -> None: self._cmds = cmds @classmethod - def build(cls, path: Path) -> 'Kustomize': + def build(cls, path: Path) -> "Kustomize": """Build cluster artifacts from the specified path.""" return Kustomize(cmds=[[KUSTOMIZE_BIN, "build", str(path)]]) - def grep(self, expr: str, path: Path | None = None, invert: bool = False) -> 'Kustomize': + def grep( + self, expr: str, path: Path | None = None, invert: bool = False + ) -> "Kustomize": """Filter resources based on an expression. Example expressions: @@ -56,14 +59,14 @@ def grep(self, expr: str, path: Path | None = None, invert: bool = False) -> 'Ku if invert: out.append("--invert-match") if path: - out.append(str(path)) + out.append(str(path)) return Kustomize(self._cmds + [out]) async def run(self) -> str: """Run the kustomize command and return the output as a string.""" return await command.run_piped(self._cmds) - async def _docs(self) -> Generator[dict[str, Any], None, None]: + async def _docs(self) -> AsyncGenerator[dict[str, Any], None]: """Run the kustomize command and return the result documents.""" out = await self.run() for doc in yaml.safe_load_all(out): @@ -71,4 +74,4 @@ async def _docs(self) -> Generator[dict[str, Any], None, None]: async def objects(self) -> list[dict[str, Any]]: """Run the kustomize command and return the result cluster objects as a list.""" - return [ doc async for doc in self._docs() ] + return [doc async for doc in self._docs()] diff --git a/requirements.txt b/requirements.txt index 25c20879..5d1e724b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ pytest-asyncio==0.20.3 pytest-cov==3.0.0 python-slugify==8.0.0 PyYAML==6.0 +types-aiofiles==22.1.0.6 types-PyYAML==6.0.12.4 typing-extensions==4.4.0 wheel==0.37.1 diff --git a/script/run-in-env.sh b/script/run-in-env.sh new file mode 100755 index 00000000..271e7a4a --- /dev/null +++ b/script/run-in-env.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +set -eu + +# Activate pyenv and virtualenv if present, then run the specified command + +# pyenv, pyenv-virtualenv +if [ -s .python-version ]; then + PYENV_VERSION=$(head -n 1 .python-version) + export PYENV_VERSION +fi + +# other common virtualenvs +my_path=$(git rev-parse --show-toplevel) + +for venv in venv .venv .; do + if [ -f "${my_path}/${venv}/bin/activate" ]; then + . "${my_path}/${venv}/bin/activate" + break + fi +done + +exec "$@" diff --git a/setup.cfg b/setup.cfg index 4c6a5f98..fc1a4d10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,8 @@ exclude = flux_local = py.typed [flake8] +exclude = venv max-line-length = 88 ignore = - E501, # black: Line too long - W503 # black: Line break before binary operator + E501, + W503 From 249f7b3ec37a143cc066eb46ae95b4a9dbb38fa6 Mon Sep 17 00:00:00 2001 From: Allen Porter Date: Mon, 6 Feb 2023 01:07:55 +0000 Subject: [PATCH 4/4] Address codespell flags --- flux_local/helm.py | 2 +- flux_local/manifest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flux_local/helm.py b/flux_local/helm.py index 9e96684c..2d1dbc0c 100644 --- a/flux_local/helm.py +++ b/flux_local/helm.py @@ -65,7 +65,7 @@ def __init__(self, repos: list[HelmRepository]) -> None: @property def config(self) -> dict[str, Any]: - """Return a synthetic repostory config object.""" + """Return a synthetic repository config object.""" now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) return { "apiVersion": "", diff --git a/flux_local/manifest.py b/flux_local/manifest.py index eb400a97..218b8a19 100644 --- a/flux_local/manifest.py +++ b/flux_local/manifest.py @@ -58,7 +58,7 @@ def from_doc(cls, doc: dict[str, Any]) -> "HelmChart": @property def chart_name(self) -> str: - """Identifer for the HelmChart.""" + """Identifier for the HelmChart.""" return f"{self.repo_namespace}-{self.repo_name}/{self.name}"