From 4a415944706c495f17eb2896ee152e11bdec522b Mon Sep 17 00:00:00 2001 From: Ramin Gharib Date: Tue, 2 Jan 2024 14:01:35 +0100 Subject: [PATCH] Use hash and trim long Helm release names instead of only trimming (#390) fixes https://github.com/bakdata/kpops/issues/46 --- docs/docs/user/migration-guide/v2-v3.md | 16 +++++++ .../component_handlers/helm_wrapper/utils.py | 24 ++++++---- kpops/components/base_components/helm_app.py | 9 +++- kpops/components/base_components/kafka_app.py | 38 +++++++-------- .../base_components/kafka_connector.py | 7 ++- .../helm_wrapper/test_utils.py | 48 ++++++++++--------- tests/components/test_kafka_app.py | 3 +- tests/components/test_kafka_connector.py | 19 +++++++- tests/components/test_kafka_sink_connector.py | 25 +++++----- .../components/test_kafka_source_connector.py | 27 ++++++----- tests/components/test_kubernetes_app.py | 14 +++++- tests/components/test_producer_app.py | 33 +++++++------ tests/components/test_streams_app.py | 43 +++++++++-------- tests/pipeline/test_template.py | 7 ++- 14 files changed, 193 insertions(+), 120 deletions(-) diff --git a/docs/docs/user/migration-guide/v2-v3.md b/docs/docs/user/migration-guide/v2-v3.md index ede1a29f6..38b86bea0 100644 --- a/docs/docs/user/migration-guide/v2-v3.md +++ b/docs/docs/user/migration-guide/v2-v3.md @@ -1,5 +1,21 @@ # Migrate from V2 to V3 +## [Use hash and trim long Helm release names instead of only trimming](https://github.com/bakdata/kpops/pull/390) + +KPOps handles long (more than 53 characters) Helm releases names differently. Helm will not find your (long) old release names anymore. Therefore, it is recommended that you should once destroy your pipeline with KPOps v2 to remove old Helm release names. After a clean destroy, re-deploy your pipeline with the KPOps v3. + +For example if you have a component with the Helm release name `example-component-name-too-long-fake-fakefakefakefakefake`. The new release name will shorten the original name to 52 characters and then replace the last 6 characters of the trimmed name with the first 5 characters of the result of SHA-1(helm_release_name). + + + +```console +example-component-name-too-long-fake-fakefakef-0a7fc ----> 52 chars +---------------------------------------------- ----- + ^Shortened helm_release_name ^first 5 characters of SHA1(helm_release_name) +``` + + + ## [Create HelmApp component](https://github.com/bakdata/kpops/pull/370) All Helm-specific parts of the built-in [`KubernetesApp`](../core-concepts/components/kubernetes-app.md) have been extracted to a new child component that is more appropriately named [`HelmApp`](../core-concepts/components/helm-app.md). It has to be renamed in your existing pipeline defintions and custom components module. diff --git a/kpops/component_handlers/helm_wrapper/utils.py b/kpops/component_handlers/helm_wrapper/utils.py index 7ad76b93a..5f5577842 100644 --- a/kpops/component_handlers/helm_wrapper/utils.py +++ b/kpops/component_handlers/helm_wrapper/utils.py @@ -1,22 +1,30 @@ +import hashlib import logging log = logging.getLogger("HelmUtils") - +ENCODING = "utf-8" RELEASE_NAME_MAX_LEN = 52 -def trim_release_name(name: str, suffix: str = "") -> str: - """Trim Helm release name while preserving suffix. +def create_helm_release_name(name: str, suffix: str = "") -> str: + """Shortens the long Helm release name. + + Creates a 52 character long release name if the name length exceeds the Helm release character length. + It first trims the string and fetches the first RELEASE_NAME_MAX_LEN - len(suffix) characters. + Then it replaces the last 6 characters with the SHA-1 encoded string (with "-") to avoid collision + and append the suffix if given. - :param name: The release name including optional suffix + :param name: The Helm release name to be shortened. :param suffix: The release suffix to preserve - :return: Truncated release name. + :return: Trimmed + hashed version of the release name if it exceeds the Helm release character length otherwise the actual release name """ if len(name) > RELEASE_NAME_MAX_LEN: - new_name = name[: (RELEASE_NAME_MAX_LEN - len(suffix))] + suffix + exact_name = name[: RELEASE_NAME_MAX_LEN - len(suffix)] + hash_name = hashlib.sha1(name.encode(ENCODING)).hexdigest() + new_name = exact_name[:-6] + "-" + hash_name[:5] + suffix log.critical( - f"Invalid Helm release name '{name}'. Truncating to {RELEASE_NAME_MAX_LEN} characters: \n {name} --> {new_name}" + f"Invalid Helm release name '{name}'. Truncating and hashing the release name: \n {name} --> {new_name}" ) - name = new_name + return new_name return name diff --git a/kpops/components/base_components/helm_app.py b/kpops/components/base_components/helm_app.py index 5d70bacfd..6fe6293c0 100644 --- a/kpops/components/base_components/helm_app.py +++ b/kpops/components/base_components/helm_app.py @@ -16,6 +16,7 @@ HelmTemplateFlags, HelmUpgradeInstallFlags, ) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components.base_components.kubernetes_app import KubernetesApp from kpops.utils.colorify import magentaify from kpops.utils.docstring import describe_attr @@ -67,7 +68,13 @@ def dry_run_handler(self) -> DryRunHandler: @property def helm_release_name(self) -> str: """The name for the Helm release. Can be overridden.""" - return self.full_name + return create_helm_release_name(self.full_name) + + @property + def clean_release_name(self) -> str: + """The name for the Helm release for cleanup jobs. Can be overridden.""" + suffix = "-clean" + return create_helm_release_name(self.helm_release_name, suffix) @property def helm_chart(self) -> str: diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index b62e54bab..0346f70b0 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -10,9 +10,10 @@ HelmRepoConfig, HelmUpgradeInstallFlags, ) -from kpops.component_handlers.helm_wrapper.utils import trim_release_name from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kubernetes_app import KubernetesAppConfig +from kpops.components.base_components.kubernetes_app import ( + KubernetesAppConfig, +) from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import CamelCaseConfigModel, DescConfigModel @@ -40,14 +41,16 @@ class KafkaAppConfig(KubernetesAppConfig): """Settings specific to Kafka Apps. :param streams: Kafka streams config - :param name_override: Override name with this value, defaults to None + :param name_override: Override name with this value """ streams: KafkaStreamsConfig = Field( default=..., description=describe_attr("streams", __doc__) ) name_override: str | None = Field( - default=None, description=describe_attr("name_override", __doc__) + default=None, + title="Nameoverride", + description=describe_attr("name_override", __doc__), ) @@ -108,28 +111,21 @@ def _run_clean_up_job( :param values: The value YAML for the chart :param dry_run: Dry run command :param retain_clean_jobs: Whether to retain the cleanup job, defaults to False - :return: """ - suffix = "-clean" - clean_up_release_name = trim_release_name( - self.helm_release_name + suffix, suffix - ) - log.info(f"Uninstall old cleanup job for {clean_up_release_name}") + log.info(f"Uninstall old cleanup job for {self.clean_release_name}") - self.__uninstall_clean_up_job(clean_up_release_name, dry_run) + self.__uninstall_clean_up_job(self.clean_release_name, dry_run) - log.info(f"Init cleanup job for {clean_up_release_name}") + log.info(f"Init cleanup job for {self.clean_release_name}") - stdout = self.__install_clean_up_job( - clean_up_release_name, suffix, values, dry_run - ) + stdout = self.__install_clean_up_job(self.clean_release_name, values, dry_run) if dry_run: - self.dry_run_handler.print_helm_diff(stdout, clean_up_release_name, log) + self.dry_run_handler.print_helm_diff(stdout, self.clean_release_name, log) if not retain_clean_jobs: - log.info(f"Uninstall cleanup job for {clean_up_release_name}") - self.__uninstall_clean_up_job(clean_up_release_name, dry_run) + log.info(f"Uninstall cleanup job for {self.clean_release_name}") + self.__uninstall_clean_up_job(self.clean_release_name, dry_run) def __uninstall_clean_up_job(self, release_name: str, dry_run: bool) -> None: """Uninstall clean up job. @@ -142,7 +138,6 @@ def __uninstall_clean_up_job(self, release_name: str, dry_run: bool) -> None: def __install_clean_up_job( self, release_name: str, - suffix: str, values: dict, dry_run: bool, ) -> str: @@ -152,11 +147,10 @@ def __install_clean_up_job( :param suffix: Suffix to add to the release name, e.g. "-clean" :param values: The Helm values for the chart :param dry_run: Whether to do a dry run of the command - :return: Install clean up job with helm, return the output of the installation + :return: Return the output of the installation """ - clean_up_release_name = trim_release_name(release_name, suffix) return self.helm.upgrade_install( - clean_up_release_name, + release_name, self.clean_up_helm_chart, dry_run, self.namespace, diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py index 7af2c5ae4..c6f21612d 100644 --- a/kpops/components/base_components/kafka_connector.py +++ b/kpops/components/base_components/kafka_connector.py @@ -17,7 +17,7 @@ HelmTemplateFlags, HelmUpgradeInstallFlags, ) -from kpops.component_handlers.helm_wrapper.utils import trim_release_name +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.component_handlers.kafka_connect.model import ( KafkaConnectorConfig, KafkaConnectorType, @@ -104,8 +104,7 @@ def helm(self) -> Helm: @property def _resetter_release_name(self) -> str: suffix = "-clean" - clean_up_release_name = self.full_name + suffix - return trim_release_name(clean_up_release_name, suffix) + return create_helm_release_name(self.full_name + suffix, suffix) @property def _resetter_helm_chart(self) -> str: @@ -244,7 +243,7 @@ def _get_kafka_connect_resetter_values( **kwargs, ), connector_type=self._connector_type.value, - name_override=self.full_name, + name_override=self.full_name + "-clean", ).model_dump(), **self.resetter_values, } diff --git a/tests/component_handlers/helm_wrapper/test_utils.py b/tests/component_handlers/helm_wrapper/test_utils.py index eef6ca14f..4a7111d88 100644 --- a/tests/component_handlers/helm_wrapper/test_utils.py +++ b/tests/component_handlers/helm_wrapper/test_utils.py @@ -1,29 +1,33 @@ -from kpops.component_handlers.helm_wrapper.utils import trim_release_name +from kpops.component_handlers.helm_wrapper.utils import ( + create_helm_release_name, +) -def test_trim_release_name_with_suffix(): - name = trim_release_name( - "example-component-name-too-long-fake-fakefakefakefakefake-clean", - suffix="-clean", - ) - assert name == "example-component-name-too-long-fake-fakefakef-clean" - assert len(name) == 52 +def test_helm_release_name_for_long_names(): + long_release_name = "example-component-name-too-long-fake-fakefakefakefakefake" + actual_release_name = create_helm_release_name(long_release_name) -def test_trim_release_name_without_suffix(): - name = trim_release_name( - "example-component-name-too-long-fake-fakefakefakefakefake" - ) - assert name == "example-component-name-too-long-fake-fakefakefakefak" - assert len(name) == 52 + expected_helm_release_name = "example-component-name-too-long-fake-fakefakef-0a7fc" + assert expected_helm_release_name == actual_release_name + assert len(expected_helm_release_name) == 52 -def test_no_trim_release_name(): - assert ( - trim_release_name("normal-name-with-no-need-of-trim-clean", suffix="-clean") - == "normal-name-with-no-need-of-trim-clean" - ) - assert ( - trim_release_name("normal-name-with-no-need-of-trim") - == "normal-name-with-no-need-of-trim" +def test_helm_release_name_for_install_and_clean_must_be_different(): + long_release_name = "example-component-name-too-long-fake-fakefakefakefakefake" + + helm_clean_release_name = create_helm_release_name(long_release_name, "-clean") + expected_helm_release_name = ( + "example-component-name-too-long-fake-fakefakef-0a7fc-clean" ) + + assert expected_helm_release_name != helm_clean_release_name + + +def test_helm_release_name_for_short_names(): + short_release_name = "example-component-name" + + actual_helm_release_name = create_helm_release_name(short_release_name) + + assert actual_helm_release_name == short_release_name + assert len(actual_helm_release_name) < 53 diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index 06af5d4f5..6a5ed7d18 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -10,6 +10,7 @@ HelmRepoConfig, HelmUpgradeInstallFlags, ) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components.base_components import KafkaApp from kpops.config import KpopsConfig @@ -92,7 +93,7 @@ def test_should_deploy_kafka_app( print_helm_diff.assert_called_once() helm_upgrade_install.assert_called_once_with( - "${pipeline_name}-example-name", + create_helm_release_name("${pipeline_name}-example-name"), "test/test-chart", True, "test-namespace", diff --git a/tests/components/test_kafka_connector.py b/tests/components/test_kafka_connector.py index 8ea178ef2..6c0e0dcc3 100644 --- a/tests/components/test_kafka_connector.py +++ b/tests/components/test_kafka_connector.py @@ -14,7 +14,8 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" CONNECTOR_NAME = "test-connector-with-long-name-0123456789abcdefghijklmnop" CONNECTOR_FULL_NAME = "${pipeline_name}-" + CONNECTOR_NAME -CONNECTOR_CLEAN_FULL_NAME = "${pipeline_name}-test-connector-with-long-name-clean" +CONNECTOR_CLEAN_FULL_NAME = CONNECTOR_FULL_NAME + "-clean" +CONNECTOR_CLEAN_RELEASE_NAME = "${pipeline_name}-test-connector-with-lon-449ec-clean" CONNECTOR_CLASS = "com.bakdata.connect.TestConnector" @@ -111,3 +112,19 @@ def test_connector_config_name_override( app={"connector.class": CONNECTOR_CLASS, "name": ""}, # type: ignore[reportGeneralTypeIssues] namespace="test-namespace", ) + + def test_resetter_release_name( + self, + config: KpopsConfig, + handlers: ComponentHandlers, + connector_config: KafkaConnectorConfig, + ): + connector = KafkaConnector( + name=CONNECTOR_NAME, + config=config, + handlers=handlers, + app=connector_config, + namespace="test-namespace", + ) + assert connector.app.name == CONNECTOR_FULL_NAME + assert connector._resetter_release_name == CONNECTOR_CLEAN_RELEASE_NAME diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index 25fa67498..81b5049d9 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -28,6 +28,7 @@ from kpops.utils.colorify import magentaify from tests.components.test_kafka_connector import ( CONNECTOR_CLEAN_FULL_NAME, + CONNECTOR_CLEAN_RELEASE_NAME, CONNECTOR_FULL_NAME, CONNECTOR_NAME, TestKafkaConnector, @@ -211,11 +212,11 @@ def test_reset_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=dry_run, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=dry_run, @@ -231,12 +232,12 @@ def test_reset_when_dry_run_is_false( "connector": CONNECTOR_FULL_NAME, "deleteConsumerGroup": False, }, - "nameOverride": CONNECTOR_FULL_NAME, + "nameOverride": CONNECTOR_CLEAN_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=dry_run, ), ] @@ -301,11 +302,11 @@ def test_clean_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=dry_run, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=dry_run, @@ -321,12 +322,12 @@ def test_clean_when_dry_run_is_false( "connector": CONNECTOR_FULL_NAME, "deleteConsumerGroup": True, }, - "nameOverride": CONNECTOR_FULL_NAME, + "nameOverride": CONNECTOR_CLEAN_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=dry_run, ), ] @@ -395,11 +396,11 @@ def test_clean_without_to_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=dry_run, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=dry_run, @@ -415,12 +416,12 @@ def test_clean_without_to_when_dry_run_is_false( "connector": CONNECTOR_FULL_NAME, "deleteConsumerGroup": True, }, - "nameOverride": CONNECTOR_FULL_NAME, + "nameOverride": CONNECTOR_CLEAN_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=dry_run, ), ] diff --git a/tests/components/test_kafka_source_connector.py b/tests/components/test_kafka_source_connector.py index 5d129e987..a34efc364 100644 --- a/tests/components/test_kafka_source_connector.py +++ b/tests/components/test_kafka_source_connector.py @@ -25,11 +25,14 @@ from kpops.utils.environment import ENV from tests.components.test_kafka_connector import ( CONNECTOR_CLEAN_FULL_NAME, + CONNECTOR_CLEAN_RELEASE_NAME, CONNECTOR_FULL_NAME, CONNECTOR_NAME, TestKafkaConnector, ) +CLEAN_SUFFIX = "-clean" + class TestKafkaSourceConnector(TestKafkaConnector): @pytest.fixture() @@ -157,11 +160,11 @@ def test_reset_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=False, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=False, @@ -177,12 +180,12 @@ def test_reset_when_dry_run_is_false( "connector": CONNECTOR_FULL_NAME, "offsetTopic": "kafka-connect-offsets", }, - "nameOverride": CONNECTOR_FULL_NAME, + "nameOverride": CONNECTOR_CLEAN_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=False, ), ] @@ -232,11 +235,11 @@ def test_clean_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=False, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=False, @@ -252,12 +255,12 @@ def test_clean_when_dry_run_is_false( "connector": CONNECTOR_FULL_NAME, "offsetTopic": "kafka-connect-offsets", }, - "nameOverride": CONNECTOR_FULL_NAME, + "nameOverride": CONNECTOR_CLEAN_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=False, ), ] @@ -307,11 +310,11 @@ def test_clean_without_to_when_dry_run_is_false( ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=False, ), mocker.call.helm.upgrade_install( - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, namespace="test-namespace", chart="bakdata-kafka-connect-resetter/kafka-connect-resetter", dry_run=False, @@ -327,12 +330,12 @@ def test_clean_without_to_when_dry_run_is_false( "connector": CONNECTOR_FULL_NAME, "offsetTopic": "kafka-connect-offsets", }, - "nameOverride": CONNECTOR_FULL_NAME, + "nameOverride": CONNECTOR_CLEAN_FULL_NAME, }, ), mocker.call.helm.uninstall( namespace="test-namespace", - release_name=CONNECTOR_CLEAN_FULL_NAME, + release_name=CONNECTOR_CLEAN_RELEASE_NAME, dry_run=False, ), ] diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index 95ab11f6c..45f575fb8 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -5,17 +5,23 @@ from pytest_mock import MockerFixture from kpops.component_handlers import ComponentHandlers +from kpops.component_handlers.helm_wrapper.model import ( + HelmRepoConfig, +) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components.base_components.kubernetes_app import ( KubernetesApp, KubernetesAppConfig, ) from kpops.config import KpopsConfig +HELM_RELEASE_NAME = create_helm_release_name("${pipeline_name}-test-kubernetes-app") + DEFAULTS_PATH = Path(__file__).parent / "resources" class KubernetesTestValue(KubernetesAppConfig): - foo: str + name_override: str class TestKubernetesApp: @@ -37,7 +43,11 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock: @pytest.fixture() def app_value(self) -> KubernetesTestValue: - return KubernetesTestValue(foo="foo") + return KubernetesTestValue(**{"name_override": "test-value"}) + + @pytest.fixture() + def repo_config(self) -> HelmRepoConfig: + return HelmRepoConfig(repository_name="test", url="https://bakdata.com") @pytest.fixture() def kubernetes_app( diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 89ca25bdd..ca8b67fac 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -7,6 +7,7 @@ from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.helm_wrapper.model import HelmUpgradeInstallFlags +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components import ProducerApp from kpops.components.base_components.models.to_section import ( OutputTopicTypes, @@ -16,11 +17,15 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" +PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" +PRODUCER_APP_FULL_NAME = "${pipeline_name}-" + PRODUCER_APP_NAME +PRODUCER_APP_RELEASE_NAME = create_helm_release_name(PRODUCER_APP_FULL_NAME) +PRODUCER_APP_CLEAN_RELEASE_NAME = create_helm_release_name( + PRODUCER_APP_RELEASE_NAME, "-clean" +) -class TestProducerApp: - PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" - PRODUCER_APP_CLEAN_NAME = "test-producer-app-with-long-n-clean" +class TestProducerApp: @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( @@ -44,7 +49,7 @@ def producer_app( self, config: KpopsConfig, handlers: ComponentHandlers ) -> ProducerApp: return ProducerApp( - name=self.PRODUCER_APP_NAME, + name=PRODUCER_APP_NAME, config=config, handlers=handlers, **{ @@ -66,7 +71,7 @@ def producer_app( def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers): producer_app = ProducerApp( - name=self.PRODUCER_APP_NAME, + name=PRODUCER_APP_NAME, config=config, handlers=handlers, **{ @@ -115,7 +120,7 @@ def test_deploy_order_when_dry_run_is_false( assert mock.mock_calls == [ mocker.call.mock_create_topics(to_section=producer_app.to, dry_run=False), mocker.call.mock_helm_upgrade_install( - "${pipeline_name}-" + self.PRODUCER_APP_NAME, + PRODUCER_APP_RELEASE_NAME, "bakdata-streams-bootstrap/producer-app", False, "test-namespace", @@ -149,7 +154,7 @@ def test_destroy( producer_app.destroy(dry_run=True) mock_helm_uninstall.assert_called_once_with( - "test-namespace", "${pipeline_name}-" + self.PRODUCER_APP_NAME, True + "test-namespace", PRODUCER_APP_RELEASE_NAME, True ) def test_should_not_reset_producer_app( @@ -175,11 +180,11 @@ def test_should_not_reset_producer_app( assert mock.mock_calls == [ mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, True, ), mocker.call.helm_upgrade_install( - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, "bakdata-streams-bootstrap/producer-app-cleanup-job", True, "test-namespace", @@ -193,12 +198,12 @@ def test_should_not_reset_producer_app( ), mocker.call.print_helm_diff( ANY, - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, logging.getLogger("KafkaApp"), ), mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, True, ), ] @@ -220,11 +225,11 @@ def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_w assert mock.mock_calls == [ mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, False, ), mocker.call.helm_upgrade_install( - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, "bakdata-streams-bootstrap/producer-app-cleanup-job", False, "test-namespace", @@ -238,7 +243,7 @@ def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clean_up_w ), mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.PRODUCER_APP_CLEAN_NAME, + PRODUCER_APP_CLEAN_RELEASE_NAME, False, ), ] diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index 93f6022f2..8943b64a3 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -9,6 +9,7 @@ HelmDiffConfig, HelmUpgradeInstallFlags, ) +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name from kpops.components import StreamsApp from kpops.components.base_components.models import TopicName from kpops.components.base_components.models.to_section import ( @@ -20,11 +21,15 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" +STREAMS_APP_NAME = "test-streams-app-with-long-name-0123456789abcdefghijklmnop" +STREAMS_APP_FULL_NAME = "${pipeline_name}-" + STREAMS_APP_NAME +STREAMS_APP_RELEASE_NAME = create_helm_release_name(STREAMS_APP_FULL_NAME) +STREAMS_APP_CLEAN_RELEASE_NAME = create_helm_release_name( + STREAMS_APP_RELEASE_NAME, "-clean" +) -class TestStreamsApp: - STREAMS_APP_NAME = "test-streams-app-with-long-name-0123456789abcdefghijklmnop" - STREAMS_APP_CLEAN_NAME = "test-streams-app-with-long-na-clean" +class TestStreamsApp: @pytest.fixture() def handlers(self) -> ComponentHandlers: return ComponentHandlers( @@ -49,7 +54,7 @@ def streams_app( self, config: KpopsConfig, handlers: ComponentHandlers ) -> StreamsApp: return StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -69,7 +74,7 @@ def streams_app( def test_set_topics(self, config: KpopsConfig, handlers: ComponentHandlers): streams_app = StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -115,7 +120,7 @@ def test_no_empty_input_topic( self, config: KpopsConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -148,7 +153,7 @@ def test_should_validate(self, config: KpopsConfig, handlers: ComponentHandlers) ValueError, match="Define role only if `type` is `pattern` or `None`" ): StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -172,7 +177,7 @@ def test_should_validate(self, config: KpopsConfig, handlers: ComponentHandlers) ValueError, match="Define `role` only if `type` is undefined" ): StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -195,7 +200,7 @@ def test_set_streams_output_from_to( self, config: KpopsConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -234,7 +239,7 @@ def test_weave_inputs_from_prev_component( self, config: KpopsConfig, handlers: ComponentHandlers ): streams_app = StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -273,7 +278,7 @@ def test_deploy_order_when_dry_run_is_false( mocker: MockerFixture, ): streams_app = StreamsApp( - name=self.STREAMS_APP_NAME, + name=STREAMS_APP_NAME, config=config, handlers=handlers, **{ @@ -318,7 +323,7 @@ def test_deploy_order_when_dry_run_is_false( assert mock.mock_calls == [ mocker.call.mock_create_topics(to_section=streams_app.to, dry_run=dry_run), mocker.call.mock_helm_upgrade_install( - "${pipeline_name}-" + self.STREAMS_APP_NAME, + STREAMS_APP_RELEASE_NAME, "bakdata-streams-bootstrap/streams-app", dry_run, "test-namespace", @@ -354,7 +359,7 @@ def test_destroy(self, streams_app: StreamsApp, mocker: MockerFixture): streams_app.destroy(dry_run=True) mock_helm_uninstall.assert_called_once_with( - "test-namespace", "${pipeline_name}-" + self.STREAMS_APP_NAME, True + "test-namespace", STREAMS_APP_RELEASE_NAME, True ) def test_reset_when_dry_run_is_false( @@ -375,11 +380,11 @@ def test_reset_when_dry_run_is_false( assert mock.mock_calls == [ mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + STREAMS_APP_CLEAN_RELEASE_NAME, dry_run, ), mocker.call.helm_upgrade_install( - "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + STREAMS_APP_CLEAN_RELEASE_NAME, "bakdata-streams-bootstrap/streams-app-cleanup-job", dry_run, "test-namespace", @@ -394,7 +399,7 @@ def test_reset_when_dry_run_is_false( ), mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + STREAMS_APP_CLEAN_RELEASE_NAME, dry_run, ), ] @@ -419,11 +424,11 @@ def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( assert mock.mock_calls == [ mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + STREAMS_APP_CLEAN_RELEASE_NAME, dry_run, ), mocker.call.helm_upgrade_install( - "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + STREAMS_APP_CLEAN_RELEASE_NAME, "bakdata-streams-bootstrap/streams-app-cleanup-job", dry_run, "test-namespace", @@ -438,7 +443,7 @@ def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean_up( ), mocker.call.helm_uninstall( "test-namespace", - "${pipeline_name}-" + self.STREAMS_APP_CLEAN_NAME, + STREAMS_APP_CLEAN_RELEASE_NAME, dry_run, ), ] diff --git a/tests/pipeline/test_template.py b/tests/pipeline/test_template.py index efd332bb6..c4b0757bb 100644 --- a/tests/pipeline/test_template.py +++ b/tests/pipeline/test_template.py @@ -7,6 +7,9 @@ from kpops.cli.main import app from kpops.component_handlers.helm_wrapper.helm import Helm +from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name + +HELM_RELEASE_NAME = create_helm_release_name("resources-custom-config-app2") runner = CliRunner() @@ -39,7 +42,7 @@ def test_default_template_config(self, run_command: MagicMock): [ "helm", "template", - "resources-custom-config-app2", + HELM_RELEASE_NAME, "bakdata-streams-bootstrap/streams-app", "--namespace", "development-namespace", @@ -78,7 +81,7 @@ def test_template_config_with_flags(self, run_command: MagicMock): [ "helm", "template", - "resources-custom-config-app2", + HELM_RELEASE_NAME, "bakdata-streams-bootstrap/streams-app", "--namespace", "development-namespace",