From c256a03679ba3bcf29de62926125608436e9f089 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Tue, 2 Jan 2024 18:24:09 +0100 Subject: [PATCH] Refactor Helm `nameOverride` (#397) Fixes #327 Remove previous workaround for setting Helm app `nameOverride` in favor of a more robust implementation in `HelmApp` --- docs/docs/schema/pipeline.json | 104 ++++++++++-------- .../component_handlers/kafka_connect/model.py | 15 +-- kpops/components/base_components/helm_app.py | 40 ++++++- kpops/components/base_components/kafka_app.py | 15 +-- .../base_components/kafka_connector.py | 10 +- .../base_components/kubernetes_app.py | 4 +- .../streams_bootstrap/producer/model.py | 4 +- .../producer/producer_app.py | 4 +- .../streams_bootstrap/streams/model.py | 4 +- .../streams_bootstrap/streams/streams_app.py | 4 +- kpops/pipeline.py | 10 -- tests/components/test_helm_app.py | 38 ++++--- tests/components/test_kafka_app.py | 1 + tests/components/test_kubernetes_app.py | 22 ++-- tests/components/test_producer_app.py | 9 +- tests/components/test_streams_app.py | 11 +- tests/pipeline/snapshots/snap_test_example.py | 5 - .../pipeline/snapshots/snap_test_pipeline.py | 37 ------- 18 files changed, 166 insertions(+), 171 deletions(-) diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index 0882ccfa5..77198a215 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -66,10 +66,10 @@ "app": { "allOf": [ { - "$ref": "#/$defs/KubernetesAppConfig" + "$ref": "#/$defs/HelmAppValues" } ], - "description": "Application-specific settings" + "description": "Helm app values" }, "from": { "anyOf": [ @@ -146,6 +146,27 @@ "title": "HelmApp", "type": "object" }, + "HelmAppValues": { + "additionalProperties": true, + "description": "Helm app values.", + "properties": { + "nameOverride": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Override name with this value", + "title": "Nameoverride" + } + }, + "title": "HelmAppValues", + "type": "object" + }, "HelmRepoConfig": { "description": "Helm repository configuration.", "properties": { @@ -420,13 +441,6 @@ "title": "KafkaSourceConnector", "type": "object" }, - "KubernetesAppConfig": { - "additionalProperties": true, - "description": "Settings specific to Kubernetes apps.", - "properties": {}, - "title": "KubernetesAppConfig", - "type": "object" - }, "OutputTopicTypes": { "description": "Types of output topic.\n\nOUTPUT (output topic), ERROR (error topic)", "enum": [ @@ -443,7 +457,7 @@ "app": { "allOf": [ { - "$ref": "#/$defs/ProducerValues" + "$ref": "#/$defs/ProducerAppValues" } ], "description": "Application-specific settings" @@ -523,6 +537,38 @@ "title": "ProducerApp", "type": "object" }, + "ProducerAppValues": { + "additionalProperties": true, + "description": "Settings specific to producers.", + "properties": { + "nameOverride": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Override name with this value", + "title": "Nameoverride" + }, + "streams": { + "allOf": [ + { + "$ref": "#/$defs/ProducerStreamsConfig" + } + ], + "description": "Kafka Streams settings" + } + }, + "required": [ + "streams" + ], + "title": "ProducerAppValues", + "type": "object" + }, "ProducerStreamsConfig": { "additionalProperties": true, "description": "Kafka Streams settings specific to Producer.", @@ -574,38 +620,6 @@ "title": "ProducerStreamsConfig", "type": "object" }, - "ProducerValues": { - "additionalProperties": true, - "description": "Settings specific to producers.", - "properties": { - "nameOverride": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Override name with this value", - "title": "Nameoverride" - }, - "streams": { - "allOf": [ - { - "$ref": "#/$defs/ProducerStreamsConfig" - } - ], - "description": "Kafka Streams settings" - } - }, - "required": [ - "streams" - ], - "title": "ProducerValues", - "type": "object" - }, "RepoAuthFlags": { "description": "Authorisation-related flags for `helm repo`.", "properties": { @@ -680,7 +694,7 @@ "app": { "allOf": [ { - "$ref": "#/$defs/StreamsAppConfig" + "$ref": "#/$defs/StreamsAppValues" } ], "description": "Application-specific settings" @@ -847,7 +861,7 @@ "title": "StreamsAppAutoScaling", "type": "object" }, - "StreamsAppConfig": { + "StreamsAppValues": { "additionalProperties": true, "description": "StreamsBoostrap app configurations.\nThe attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.", "properties": { @@ -888,7 +902,7 @@ "required": [ "streams" ], - "title": "StreamsAppConfig", + "title": "StreamsAppValues", "type": "object" }, "StreamsConfig": { diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py index a7ec45af9..59cdba7b9 100644 --- a/kpops/component_handlers/kafka_connect/model.py +++ b/kpops/component_handlers/kafka_connect/model.py @@ -11,6 +11,7 @@ from pydantic.json_schema import SkipJsonSchema from typing_extensions import override +from kpops.components.base_components.helm_app import HelmAppValues from kpops.utils.pydantic import ( CamelCaseConfigModel, DescConfigModel, @@ -92,21 +93,13 @@ class KafkaConnectConfigErrorResponse(BaseModel): configs: list[KafkaConnectConfigDescription] -class KafkaConnectResetterConfig(CamelCaseConfigModel): +class KafkaConnectorResetterConfig(CamelCaseConfigModel): brokers: str connector: str delete_consumer_group: bool | None = None offset_topic: str | None = None -class KafkaConnectResetterValues(CamelCaseConfigModel): +class KafkaConnectorResetterValues(HelmAppValues): connector_type: Literal["source", "sink"] - config: KafkaConnectResetterConfig - name_override: str - - # TODO(Ivan Yordanov): Replace with a function decorated with `@model_serializer` - # BEWARE! All default values are enforced, hard to replicate without - # access to ``model_dump`` - @override - def model_dump(self, **_) -> dict[str, Any]: - return super().model_dump(by_alias=True, exclude_none=True) + config: KafkaConnectorResetterConfig diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 6fe6293c0..36e3bd21b 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -17,7 +17,10 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.base_components.kubernetes_app import KubernetesApp +from kpops.components.base_components.kubernetes_app import ( + KubernetesApp, + KubernetesAppValues, +) from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import exclude_by_name @@ -25,6 +28,28 @@ log = logging.getLogger("HelmApp") +class HelmAppValues(KubernetesAppValues): + """Helm app values. + + :param name_override: Override name with this value + """ + + name_override: str | None = Field( + default=None, + title="Nameoverride", + description=describe_attr("name_override", __doc__), + ) + + # TODO(Ivan Yordanov): Replace with a function decorated with `@model_serializer` + # BEWARE! All default values are enforced, hard to replicate without + # access to ``model_dump`` + @override + def model_dump(self, **_) -> dict[str, Any]: + return super().model_dump( + by_alias=True, exclude_none=True, exclude_defaults=True + ) + + class HelmApp(KubernetesApp): """Kubernetes app managed through Helm with an associated Helm chart. @@ -32,6 +57,7 @@ class HelmApp(KubernetesApp): deploying the component, defaults to None this means that the command "helm repo add" is not called and Helm expects a path to local Helm chart. :param version: Helm chart version, defaults to None + :param app: Helm app values """ repo_config: HelmRepoConfig | None = Field( @@ -42,6 +68,10 @@ class HelmApp(KubernetesApp): default=None, description=describe_attr("version", __doc__), ) + app: HelmAppValues = Field( + default=..., + description=describe_attr("app", __doc__), + ) @cached_property def helm(self) -> Helm: @@ -74,7 +104,7 @@ def helm_release_name(self) -> str: def clean_release_name(self) -> str: """The name for the Helm release for cleanup jobs. Can be overridden.""" suffix = "-clean" - return create_helm_release_name(self.helm_release_name, suffix) + return create_helm_release_name(self.full_name + suffix, suffix) @property def helm_chart(self) -> str: @@ -149,9 +179,9 @@ def to_helm_values(self) -> dict: :returns: Thte values to be used by Helm """ - return self.app.model_dump( - by_alias=True, exclude_none=True, exclude_defaults=True - ) + if self.app.name_override is None: + self.app.name_override = self.full_name + return self.app.model_dump() def print_helm_diff(self, stdout: str) -> None: """Print the diff of the last and current release of this component. diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index 0346f70b0..c7c983e0d 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -10,10 +10,7 @@ HelmRepoConfig, HelmUpgradeInstallFlags, ) -from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kubernetes_app import ( - KubernetesAppConfig, -) +from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel, DescConfigModel @@ -37,21 +34,15 @@ class KafkaStreamsConfig(CamelCaseConfigModel, DescConfigModel): ) -class KafkaAppConfig(KubernetesAppConfig): +class KafkaAppValues(HelmAppValues): """Settings specific to Kafka Apps. :param streams: Kafka streams config - :param name_override: Override name with this value """ streams: KafkaStreamsConfig = Field( default=..., description=describe_attr("streams", __doc__) ) - name_override: str | None = Field( - default=None, - title="Nameoverride", - description=describe_attr("name_override", __doc__), - ) class KafkaApp(HelmApp, ABC): @@ -66,7 +57,7 @@ class KafkaApp(HelmApp, ABC): :param version: Helm chart version, defaults to "2.9.0" """ - app: KafkaAppConfig = Field( + app: KafkaAppValues = Field( default=..., description=describe_attr("app", __doc__), ) diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index c6f21612d..22a9a4d8f 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -20,9 +20,9 @@ from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.component_handlers.kafka_connect.model import ( KafkaConnectorConfig, + KafkaConnectorResetterConfig, + KafkaConnectorResetterValues, KafkaConnectorType, - KafkaConnectResetterConfig, - KafkaConnectResetterValues, ) from kpops.components.base_components.base_defaults_component import deduplicate from kpops.components.base_components.models.from_section import FromTopic @@ -175,7 +175,7 @@ def _run_connect_resetter( :param dry_run: If the cleanup should be run in dry run mode or not :param retain_clean_jobs: If the cleanup job should be kept - :param kwargs: Other values for the KafkaConnectResetter + :param kwargs: Other values for the KafkaConnectorResetter """ log.info( magentaify( @@ -236,8 +236,8 @@ def _get_kafka_connect_resetter_values( :return: The Helm chart values of the connector resetter """ return { - **KafkaConnectResetterValues( - config=KafkaConnectResetterConfig( + **KafkaConnectorResetterValues( + config=KafkaConnectorResetterConfig( connector=self.full_name, brokers=self.config.kafka_brokers, **kwargs, diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py index cae474cee..2b4065191 100644 --- a/kpops/components/base_components/kubernetes_app.py +++ b/kpops/components/base_components/kubernetes_app.py @@ -18,7 +18,7 @@ ) -class KubernetesAppConfig(CamelCaseConfigModel, DescConfigModel): +class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel): """Settings specific to Kubernetes apps.""" model_config = ConfigDict( @@ -39,7 +39,7 @@ class KubernetesApp(PipelineComponent, ABC): default=..., description=describe_attr("namespace", __doc__), ) - app: KubernetesAppConfig = Field( + app: KubernetesAppValues = Field( default=..., description=describe_attr("app", __doc__), ) diff --git a/kpops/components/streams_bootstrap/producer/model.py b/kpops/components/streams_bootstrap/producer/model.py index 01bda1dbc..53db5af67 100644 --- a/kpops/components/streams_bootstrap/producer/model.py +++ b/kpops/components/streams_bootstrap/producer/model.py @@ -1,7 +1,7 @@ from pydantic import ConfigDict, Field from kpops.components.base_components.kafka_app import ( - KafkaAppConfig, + KafkaAppValues, KafkaStreamsConfig, ) from kpops.utils.docstring import describe_attr @@ -22,7 +22,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig): ) -class ProducerValues(KafkaAppConfig): +class ProducerAppValues(KafkaAppValues): """Settings specific to producers. :param streams: Kafka Streams settings diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index 6091cdd77..e37529bae 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -9,7 +9,7 @@ TopicConfig, ) from kpops.components.streams_bootstrap.app_type import AppType -from kpops.components.streams_bootstrap.producer.model import ProducerValues +from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr @@ -25,7 +25,7 @@ class ProducerApp(KafkaApp): :param from_: Producer doesn't support FromSection, defaults to None """ - app: ProducerValues = Field( + app: ProducerAppValues = Field( default=..., description=describe_attr("app", __doc__), ) diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py index 2c8b952ce..b52bc162c 100644 --- a/kpops/components/streams_bootstrap/streams/model.py +++ b/kpops/components/streams_bootstrap/streams/model.py @@ -5,7 +5,7 @@ from kpops.components.base_components.base_defaults_component import deduplicate from kpops.components.base_components.kafka_app import ( - KafkaAppConfig, + KafkaAppValues, KafkaStreamsConfig, ) from kpops.utils.docstring import describe_attr @@ -166,7 +166,7 @@ class StreamsAppAutoScaling(CamelCaseConfigModel, DescConfigModel): model_config = ConfigDict(extra="allow") -class StreamsAppConfig(KafkaAppConfig): +class StreamsAppValues(KafkaAppValues): """StreamsBoostrap app configurations. The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart. diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index a466b4eba..e8a434b70 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -3,7 +3,7 @@ from kpops.components.base_components.kafka_app import KafkaApp from kpops.components.streams_bootstrap.app_type import AppType -from kpops.components.streams_bootstrap.streams.model import StreamsAppConfig +from kpops.components.streams_bootstrap.streams.model import StreamsAppValues from kpops.utils.docstring import describe_attr @@ -13,7 +13,7 @@ class StreamsApp(KafkaApp): :param app: Application-specific settings """ - app: StreamsAppConfig = Field( + app: StreamsAppValues = Field( default=..., description=describe_attr("app", __doc__), ) diff --git a/kpops/pipeline.py b/kpops/pipeline.py index ad69521e1..a409f8e35 100644 --- a/kpops/pipeline.py +++ b/kpops/pipeline.py @@ -3,7 +3,6 @@ import json import logging from collections import Counter -from contextlib import suppress from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -53,7 +52,6 @@ def find(self, component_name: str) -> PipelineComponent: raise ValueError(msg) def add(self, component: PipelineComponent) -> None: - self._populate_component_name(component) self.root.append(component) def __bool__(self) -> bool: @@ -78,14 +76,6 @@ def validate_unique_names(self) -> None: msg = f"step names should be unique. duplicate step names: {', '.join(duplicates)}" raise ValidationError(msg) - @staticmethod - def _populate_component_name(component: PipelineComponent) -> None: # TODO: remove - with suppress( - AttributeError # Some components like Kafka Connect do not have a name_override attribute - ): - if (app := getattr(component, "app")) and app.name_override is None: - app.name_override = component.full_name - def create_env_components_index( environment_components: list[dict], diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index 0b933b1e9..e43c9de41 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -12,18 +12,13 @@ HelmUpgradeInstallFlags, RepoAuthFlags, ) -from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kubernetes_app import KubernetesAppConfig +from kpops.components.base_components.helm_app import HelmApp, HelmAppValues from kpops.config import KpopsConfig from kpops.utils.colorify import magentaify DEFAULTS_PATH = Path(__file__).parent / "resources" -class HelmTestValue(KubernetesAppConfig): - name_override: str - - class TestHelmApp: @pytest.fixture() def config(self) -> KpopsConfig: @@ -51,8 +46,8 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.helm_app.log.info") @pytest.fixture() - def app_value(self) -> HelmTestValue: - return HelmTestValue(name_override="test-value") + def app_values(self) -> HelmAppValues: + return HelmAppValues(**{"foo": "test-value"}) @pytest.fixture() def repo_config(self) -> HelmRepoConfig: @@ -63,14 +58,14 @@ def helm_app( self, config: KpopsConfig, handlers: ComponentHandlers, - app_value: HelmTestValue, + app_values: HelmAppValues, repo_config: HelmRepoConfig, ) -> HelmApp: return HelmApp( name="test-helm-app", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", repo_config=repo_config, ) @@ -97,7 +92,10 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( "test/test-chart", False, "test-namespace", - {"nameOverride": "test-value"}, + { + "nameOverride": "${pipeline_name}-test-helm-app", + "foo": "test-value", + }, HelmUpgradeInstallFlags(), ) @@ -107,7 +105,7 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( handlers: ComponentHandlers, helm_mock: MagicMock, mocker: MockerFixture, - app_value: HelmTestValue, + app_values: HelmAppValues, ): repo_config = HelmRepoConfig( repository_name="test-repo", url="https://test.com/charts/" @@ -116,7 +114,7 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( name="test-helm-app", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", repo_config=repo_config, version="3.4.5", @@ -142,7 +140,10 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( "test/test-chart", False, "test-namespace", - {"nameOverride": "test-value"}, + { + "nameOverride": "${pipeline_name}-test-helm-app", + "foo": "test-value", + }, HelmUpgradeInstallFlags(version="3.4.5"), ), ] @@ -152,7 +153,7 @@ def test_should_deploy_app_with_local_helm_chart( config: KpopsConfig, handlers: ComponentHandlers, helm_mock: MagicMock, - app_value: HelmTestValue, + app_values: HelmAppValues, ): class AppWithLocalChart(HelmApp): repo_config: None = None @@ -166,7 +167,7 @@ def helm_chart(self) -> str: name="test-app-with-local-chart", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", ) @@ -179,7 +180,10 @@ def helm_chart(self) -> str: "path/to/helm/charts/", False, "test-namespace", - {"nameOverride": "test-value"}, + { + "nameOverride": "${pipeline_name}-test-app-with-local-chart", + "foo": "test-value", + }, HelmUpgradeInstallFlags(), ) diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index 6a5ed7d18..d7e8fd5d4 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -98,6 +98,7 @@ def test_should_deploy_kafka_app( True, "test-namespace", { + "nameOverride": "${pipeline_name}-example-name", "streams": {"brokers": "fake-broker:9092", "outputTopic": "test"}, }, HelmUpgradeInstallFlags(version="1.2.3"), diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index 45f575fb8..ebc2701fa 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -11,7 +11,7 @@ from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components.base_components.kubernetes_app import ( KubernetesApp, - KubernetesAppConfig, + KubernetesAppValues, ) from kpops.config import KpopsConfig @@ -20,8 +20,8 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" -class KubernetesTestValue(KubernetesAppConfig): - name_override: str +class KubernetesTestValues(KubernetesAppValues): + foo: str class TestKubernetesApp: @@ -42,8 +42,8 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock: return mocker.patch("kpops.components.base_components.kubernetes_app.log.info") @pytest.fixture() - def app_value(self) -> KubernetesTestValue: - return KubernetesTestValue(**{"name_override": "test-value"}) + def app_values(self) -> KubernetesTestValues: + return KubernetesTestValues(foo="foo") @pytest.fixture() def repo_config(self) -> HelmRepoConfig: @@ -54,13 +54,13 @@ def kubernetes_app( self, config: KpopsConfig, handlers: ComponentHandlers, - app_value: KubernetesTestValue, + app_values: KubernetesTestValues, ) -> KubernetesApp: return KubernetesApp( name="test-kubernetes-app", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", ) @@ -68,7 +68,7 @@ def test_should_raise_value_error_when_name_is_not_valid( self, config: KpopsConfig, handlers: ComponentHandlers, - app_value: KubernetesTestValue, + app_values: KubernetesTestValues, ): with pytest.raises( ValueError, match=r"The component name .* is invalid for Kubernetes." @@ -77,7 +77,7 @@ def test_should_raise_value_error_when_name_is_not_valid( name="Not-Compatible*", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", ) @@ -88,7 +88,7 @@ def test_should_raise_value_error_when_name_is_not_valid( name="snake_case*", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", ) @@ -96,6 +96,6 @@ def test_should_raise_value_error_when_name_is_not_valid( name="valid-name", config=config, handlers=handlers, - app=app_value, + app=app_values, namespace="test-namespace", ) diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index ca8b67fac..07e78bb6a 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -20,12 +20,16 @@ PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" PRODUCER_APP_FULL_NAME = "${pipeline_name}-" + PRODUCER_APP_NAME PRODUCER_APP_RELEASE_NAME = create_helm_release_name(PRODUCER_APP_FULL_NAME) +PRODUCER_APP_CLEAN_FULL_NAME = PRODUCER_APP_FULL_NAME + "-clean" PRODUCER_APP_CLEAN_RELEASE_NAME = create_helm_release_name( - PRODUCER_APP_RELEASE_NAME, "-clean" + PRODUCER_APP_CLEAN_FULL_NAME, "-clean" ) class TestProducerApp: + def test_release_name(self): + assert PRODUCER_APP_CLEAN_RELEASE_NAME.endswith("-clean") + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( @@ -125,6 +129,7 @@ def test_deploy_order_when_dry_run_is_false( False, "test-namespace", { + "nameOverride": PRODUCER_APP_FULL_NAME, "streams": { "brokers": "fake-broker:9092", "outputTopic": "${output_topic_name}", @@ -189,6 +194,7 @@ def test_should_not_reset_producer_app( True, "test-namespace", { + "nameOverride": PRODUCER_APP_FULL_NAME, "streams": { "brokers": "fake-broker:9092", "outputTopic": "${output_topic_name}", @@ -234,6 +240,7 @@ def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_w False, "test-namespace", { + "nameOverride": PRODUCER_APP_FULL_NAME, "streams": { "brokers": "fake-broker:9092", "outputTopic": "${output_topic_name}", diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 8943b64a3..ebd5cf7d9 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -24,12 +24,16 @@ STREAMS_APP_NAME = "test-streams-app-with-long-name-0123456789abcdefghijklmnop" STREAMS_APP_FULL_NAME = "${pipeline_name}-" + STREAMS_APP_NAME STREAMS_APP_RELEASE_NAME = create_helm_release_name(STREAMS_APP_FULL_NAME) +STREAMS_APP_CLEAN_FULL_NAME = STREAMS_APP_FULL_NAME + "-clean" STREAMS_APP_CLEAN_RELEASE_NAME = create_helm_release_name( - STREAMS_APP_RELEASE_NAME, "-clean" + STREAMS_APP_CLEAN_FULL_NAME, "-clean" ) class TestStreamsApp: + def test_release_name(self): + assert STREAMS_APP_CLEAN_RELEASE_NAME.endswith("-clean") + @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( @@ -328,6 +332,7 @@ def test_deploy_order_when_dry_run_is_false( dry_run, "test-namespace", { + "nameOverride": STREAMS_APP_FULL_NAME, "streams": { "brokers": "fake-broker:9092", "extraOutputTopics": { @@ -336,7 +341,7 @@ def test_deploy_order_when_dry_run_is_false( }, "outputTopic": "${output_topic_name}", "errorTopic": "${error_topic_name}", - } + }, }, HelmUpgradeInstallFlags( create_namespace=False, @@ -389,6 +394,7 @@ def test_reset_when_dry_run_is_false( dry_run, "test-namespace", { + "nameOverride": STREAMS_APP_FULL_NAME, "streams": { "brokers": "fake-broker:9092", "outputTopic": "${output_topic_name}", @@ -433,6 +439,7 @@ def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( dry_run, "test-namespace", { + "nameOverride": STREAMS_APP_FULL_NAME, "streams": { "brokers": "fake-broker:9092", "outputTopic": "${output_topic_name}", diff --git a/tests/pipeline/snapshots/snap_test_example.py b/tests/pipeline/snapshots/snap_test_example.py index 14d3d650c..a88a7ee4a 100644 --- a/tests/pipeline/snapshots/snap_test_example.py +++ b/tests/pipeline/snapshots/snap_test_example.py @@ -13,7 +13,6 @@ 'debug': True, 'image': '${DOCKER_REGISTRY}/atm-demo-accountproducer', 'imageTag': '1.0.0', - 'nameOverride': 'account-producer', 'prometheus': { 'jmx': { 'enabled': False @@ -64,7 +63,6 @@ 'debug': True, 'image': '${DOCKER_REGISTRY}/atm-demo-transactionavroproducer', 'imageTag': '1.0.0', - 'nameOverride': 'transaction-avro-producer', 'prometheus': { 'jmx': { 'enabled': False @@ -120,7 +118,6 @@ 'labels': { 'pipeline': 'bakdata-atm-fraud-detection' }, - 'nameOverride': 'transaction-joiner', 'prometheus': { 'jmx': { 'enabled': False @@ -182,7 +179,6 @@ 'labels': { 'pipeline': 'bakdata-atm-fraud-detection' }, - 'nameOverride': 'fraud-detector', 'prometheus': { 'jmx': { 'enabled': False @@ -244,7 +240,6 @@ 'labels': { 'pipeline': 'bakdata-atm-fraud-detection' }, - 'nameOverride': 'account-linker', 'prometheus': { 'jmx': { 'enabled': False diff --git a/tests/pipeline/snapshots/snap_test_pipeline.py b/tests/pipeline/snapshots/snap_test_pipeline.py index c9fee4d4b..0da4f9260 100644 --- a/tests/pipeline/snapshots/snap_test_pipeline.py +++ b/tests/pipeline/snapshots/snap_test_pipeline.py @@ -10,7 +10,6 @@ snapshots['TestPipeline.test_default_config test-pipeline'] = [ { 'app': { - 'nameOverride': 'resources-custom-config-app1', 'resources': { 'limits': { 'memory': '2G' @@ -58,7 +57,6 @@ 'labels': { 'pipeline': 'resources-custom-config' }, - 'nameOverride': 'resources-custom-config-app2', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'errorTopic': 'resources-custom-config-app2-error', @@ -110,7 +108,6 @@ }, 'image': 'example-registry/fake-image', 'imageTag': '0.0.1', - 'nameOverride': 'resources-pipeline-with-inflate-scheduled-producer', 'schedule': '30 3/8 * * *', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', @@ -165,7 +162,6 @@ 'commandLine': { 'CONVERT_XML': True }, - 'nameOverride': 'resources-pipeline-with-inflate-converter', 'resources': { 'limits': { 'memory': '2G' @@ -242,7 +238,6 @@ }, 'image': 'fake-registry/filter', 'imageTag': '2.4.1', - 'nameOverride': 'resources-pipeline-with-inflate-should-inflate', 'replicaCount': 4, 'resources': { 'requests': { @@ -345,7 +340,6 @@ }, { 'app': { - 'nameOverride': 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -397,7 +391,6 @@ { 'app': { 'image': 'fake-image', - 'nameOverride': 'resources-kafka-connect-sink-streams-app', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -492,7 +485,6 @@ }, 'image': 'example-registry/fake-image', 'imageTag': '0.0.1', - 'nameOverride': 'resources-first-pipeline-scheduled-producer', 'schedule': '30 3/8 * * *', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', @@ -547,7 +539,6 @@ 'commandLine': { 'CONVERT_XML': True }, - 'nameOverride': 'resources-first-pipeline-converter', 'resources': { 'limits': { 'memory': '2G' @@ -624,7 +615,6 @@ }, 'image': 'fake-registry/filter', 'imageTag': '2.4.1', - 'nameOverride': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', 'replicaCount': 4, 'resources': { 'requests': { @@ -683,7 +673,6 @@ snapshots['TestPipeline.test_model_serialization test-pipeline'] = [ { 'app': { - 'nameOverride': 'resources-pipeline-with-paths-account-producer', 'streams': { 'brokers': 'test', 'extraOutputTopics': { @@ -716,7 +705,6 @@ 'commandLine': { 'CONVERT_XML': True }, - 'nameOverride': 'resources-no-input-topic-pipeline-app1', 'resources': { 'limits': { 'memory': '2G' @@ -779,7 +767,6 @@ }, { 'app': { - 'nameOverride': 'resources-no-input-topic-pipeline-app2', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -839,7 +826,6 @@ { 'app': { 'image': 'fake-image', - 'nameOverride': 'resources-no-user-defined-components-streams-app', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -904,7 +890,6 @@ }, 'image': 'example-registry/fake-image', 'imageTag': '0.0.1', - 'nameOverride': 'resources-pipeline-with-envs-input-producer', 'schedule': '20 3/8 * * *', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', @@ -959,7 +944,6 @@ 'commandLine': { 'CONVERT_XML': True }, - 'nameOverride': 'resources-pipeline-with-envs-converter', 'resources': { 'limits': { 'memory': '2G' @@ -1036,7 +1020,6 @@ }, 'image': 'fake-registry/filter', 'imageTag': '2.4.1', - 'nameOverride': 'resources-pipeline-with-envs-filter', 'replicaCount': 4, 'resources': { 'requests': { @@ -1098,7 +1081,6 @@ 'debug': True, 'image': '${DOCKER_REGISTRY}/atm-demo-accountproducer', 'imageTag': '1.0.0', - 'nameOverride': 'from-pipeline-component-account-producer', 'prometheus': { 'jmx': { 'enabled': False @@ -1132,7 +1114,6 @@ snapshots['TestPipeline.test_read_from_component test-pipeline'] = [ { 'app': { - 'nameOverride': 'resources-read-from-component-producer1', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'extraOutputTopics': { @@ -1167,7 +1148,6 @@ }, { 'app': { - 'nameOverride': 'producer2', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'extraOutputTopics': { @@ -1217,7 +1197,6 @@ }, 'image': 'fake-registry/filter', 'imageTag': '2.4.1', - 'nameOverride': 'resources-read-from-component-inflate-step', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1314,7 +1293,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-inflate-step-inflated-streams-app', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1377,7 +1355,6 @@ }, 'image': 'fake-registry/filter', 'imageTag': '2.4.1', - 'nameOverride': 'inflate-step-without-prefix', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1474,7 +1451,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1522,7 +1498,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-consumer1', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1579,7 +1554,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-consumer2', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1634,7 +1608,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-consumer3', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1689,7 +1662,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-consumer4', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1740,7 +1712,6 @@ }, { 'app': { - 'nameOverride': 'resources-read-from-component-consumer5', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': { @@ -1804,7 +1775,6 @@ 'app_schedule': '30 3/8 * * *', 'app_type': 'scheduled-producer' }, - 'nameOverride': 'resources-component-type-substitution-scheduled-producer', 'schedule': '30 3/8 * * *', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', @@ -1859,7 +1829,6 @@ 'commandLine': { 'CONVERT_XML': True }, - 'nameOverride': 'resources-component-type-substitution-converter', 'resources': { 'limits': { 'memory': '2G' @@ -1943,7 +1912,6 @@ 'filter': 'filter-app-filter', 'test_placeholder_in_placeholder': 'filter-app-filter' }, - 'nameOverride': 'resources-component-type-substitution-filter-app', 'replicaCount': 4, 'resources': { 'requests': { @@ -2002,7 +1970,6 @@ snapshots['TestPipeline.test_with_custom_config_with_absolute_defaults_path test-pipeline'] = [ { 'app': { - 'nameOverride': 'resources-custom-config-app1', 'resources': { 'limits': { 'memory': '2G' @@ -2050,7 +2017,6 @@ 'labels': { 'pipeline': 'resources-custom-config' }, - 'nameOverride': 'resources-custom-config-app2', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'errorTopic': 'app2-dead-letter-topic', @@ -2097,7 +2063,6 @@ snapshots['TestPipeline.test_with_custom_config_with_relative_defaults_path test-pipeline'] = [ { 'app': { - 'nameOverride': 'resources-custom-config-app1', 'resources': { 'limits': { 'memory': '2G' @@ -2145,7 +2110,6 @@ 'labels': { 'pipeline': 'resources-custom-config' }, - 'nameOverride': 'resources-custom-config-app2', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'errorTopic': 'app2-dead-letter-topic', @@ -2193,7 +2157,6 @@ { 'app': { 'image': 'fake-image', - 'nameOverride': 'resources-kafka-connect-sink-streams-app-development', 'streams': { 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', 'config': {