Use hash and trim long Helm release names instead of only trimming (#390

)

fixes #46

docs/docs/user/migration-guide/v2-v3.md

@@ -1,5 +1,21 @@
 # Migrate from V2 to V3
 
+## [Use hash and trim long Helm release names instead of only trimming](https://github.com/bakdata/kpops/pull/390)
+
+KPOps handles long (more than 53 characters) Helm releases names differently. Helm will not find your (long) old release names anymore. Therefore, it is recommended that you should once destroy your pipeline with KPOps v2 to remove old Helm release names. After a clean destroy, re-deploy your pipeline with the KPOps v3.
+
+For example if you have a component with the Helm release name `example-component-name-too-long-fake-fakefakefakefakefake`. The new release name will shorten the original name to 52 characters and then replace the last 6 characters of the trimmed name with the first 5 characters of the result of SHA-1(helm_release_name).
+
+<!-- dprint-ignore-start -->
+
+```console
+example-component-name-too-long-fake-fakefakef-0a7fc ----> 52 chars
+---------------------------------------------- -----
+  ^Shortened helm_release_name                 ^first 5 characters of SHA1(helm_release_name)
+```
+
+<!-- dprint-ignore-end -->
+
 ## [Create HelmApp component](https://github.com/bakdata/kpops/pull/370)
 
 All Helm-specific parts of the built-in [`KubernetesApp`](../core-concepts/components/kubernetes-app.md) have been extracted to a new child component that is more appropriately named [`HelmApp`](../core-concepts/components/helm-app.md). It has to be renamed in your existing pipeline defintions and custom components module.

kpops/component_handlers/helm_wrapper/utils.py

@@ -1,22 +1,30 @@
+import hashlib
 import logging
 
 log = logging.getLogger("HelmUtils")
 
-
+ENCODING = "utf-8"
 RELEASE_NAME_MAX_LEN = 52
 
 
-def trim_release_name(name: str, suffix: str = "") -> str:
-    """Trim Helm release name while preserving suffix.
+def create_helm_release_name(name: str, suffix: str = "") -> str:
+    """Shortens the long Helm release name.
+
+    Creates a 52 character long release name if the name length exceeds the Helm release character length.
+    It first trims the string and fetches the first RELEASE_NAME_MAX_LEN - len(suffix) characters.
+    Then it replaces the last 6 characters with the SHA-1 encoded string (with "-") to avoid collision
+    and append the suffix if given.
 
-    :param name: The release name including optional suffix
+    :param name: The Helm release name to be shortened.
     :param suffix: The release suffix to preserve
-    :return: Truncated release name.
+    :return: Trimmed + hashed version of the release name if it exceeds the Helm release character length otherwise the actual release name
     """
     if len(name) > RELEASE_NAME_MAX_LEN:
-        new_name = name[: (RELEASE_NAME_MAX_LEN - len(suffix))] + suffix
+        exact_name = name[: RELEASE_NAME_MAX_LEN - len(suffix)]
+        hash_name = hashlib.sha1(name.encode(ENCODING)).hexdigest()
+        new_name = exact_name[:-6] + "-" + hash_name[:5] + suffix
         log.critical(
-            f"Invalid Helm release name '{name}'. Truncating to {RELEASE_NAME_MAX_LEN} characters: \n {name} --> {new_name}"
+            f"Invalid Helm release name '{name}'. Truncating and hashing the release name: \n {name} --> {new_name}"
         )
-        name = new_name
+        return new_name
     return name

kpops/components/base_components/helm_app.py

@@ -16,6 +16,7 @@
     HelmTemplateFlags,
     HelmUpgradeInstallFlags,
 )
+from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name
 from kpops.components.base_components.kubernetes_app import KubernetesApp
 from kpops.utils.colorify import magentaify
 from kpops.utils.docstring import describe_attr
@@ -67,7 +68,13 @@ def dry_run_handler(self) -> DryRunHandler:
     @property
     def helm_release_name(self) -> str:
         """The name for the Helm release. Can be overridden."""
-        return self.full_name
+        return create_helm_release_name(self.full_name)
+
+    @property
+    def clean_release_name(self) -> str:
+        """The name for the Helm release for cleanup jobs. Can be overridden."""
+        suffix = "-clean"
+        return create_helm_release_name(self.helm_release_name, suffix)
 
     @property
     def helm_chart(self) -> str:

kpops/components/base_components/kafka_app.py

@@ -10,9 +10,10 @@
     HelmRepoConfig,
     HelmUpgradeInstallFlags,
 )
-from kpops.component_handlers.helm_wrapper.utils import trim_release_name
 from kpops.components.base_components.helm_app import HelmApp
-from kpops.components.base_components.kubernetes_app import KubernetesAppConfig
+from kpops.components.base_components.kubernetes_app import (
+    KubernetesAppConfig,
+)
 from kpops.utils.docstring import describe_attr
 from kpops.utils.pydantic import CamelCaseConfigModel, DescConfigModel
 
@@ -40,14 +41,16 @@ class KafkaAppConfig(KubernetesAppConfig):
     """Settings specific to Kafka Apps.
 
     :param streams: Kafka streams config
-    :param name_override: Override name with this value, defaults to None
+    :param name_override: Override name with this value
     """
 
     streams: KafkaStreamsConfig = Field(
         default=..., description=describe_attr("streams", __doc__)
     )
     name_override: str | None = Field(
-        default=None, description=describe_attr("name_override", __doc__)
+        default=None,
+        title="Nameoverride",
+        description=describe_attr("name_override", __doc__),
     )
 
 
@@ -108,28 +111,21 @@ def _run_clean_up_job(
         :param values: The value YAML for the chart
         :param dry_run: Dry run command
         :param retain_clean_jobs: Whether to retain the cleanup job, defaults to False
-        :return:
         """
-        suffix = "-clean"
-        clean_up_release_name = trim_release_name(
-            self.helm_release_name + suffix, suffix
-        )
-        log.info(f"Uninstall old cleanup job for {clean_up_release_name}")
+        log.info(f"Uninstall old cleanup job for {self.clean_release_name}")
 
-        self.__uninstall_clean_up_job(clean_up_release_name, dry_run)
+        self.__uninstall_clean_up_job(self.clean_release_name, dry_run)
 
-        log.info(f"Init cleanup job for {clean_up_release_name}")
+        log.info(f"Init cleanup job for {self.clean_release_name}")
 
-        stdout = self.__install_clean_up_job(
-            clean_up_release_name, suffix, values, dry_run
-        )
+        stdout = self.__install_clean_up_job(self.clean_release_name, values, dry_run)
 
         if dry_run:
-            self.dry_run_handler.print_helm_diff(stdout, clean_up_release_name, log)
+            self.dry_run_handler.print_helm_diff(stdout, self.clean_release_name, log)
 
         if not retain_clean_jobs:
-            log.info(f"Uninstall cleanup job for {clean_up_release_name}")
-            self.__uninstall_clean_up_job(clean_up_release_name, dry_run)
+            log.info(f"Uninstall cleanup job for {self.clean_release_name}")
+            self.__uninstall_clean_up_job(self.clean_release_name, dry_run)
 
     def __uninstall_clean_up_job(self, release_name: str, dry_run: bool) -> None:
         """Uninstall clean up job.
@@ -142,7 +138,6 @@ def __uninstall_clean_up_job(self, release_name: str, dry_run: bool) -> None:
     def __install_clean_up_job(
         self,
         release_name: str,
-        suffix: str,
         values: dict,
         dry_run: bool,
     ) -> str:
@@ -152,11 +147,10 @@ def __install_clean_up_job(
         :param suffix: Suffix to add to the release name, e.g. "-clean"
         :param values: The Helm values for the chart
         :param dry_run: Whether to do a dry run of the command
-        :return: Install clean up job with helm, return the output of the installation
+        :return: Return the output of the installation
         """
-        clean_up_release_name = trim_release_name(release_name, suffix)
         return self.helm.upgrade_install(
-            clean_up_release_name,
+            release_name,
             self.clean_up_helm_chart,
             dry_run,
             self.namespace,

kpops/components/base_components/kafka_connector.py

@@ -17,7 +17,7 @@
     HelmTemplateFlags,
     HelmUpgradeInstallFlags,
 )
-from kpops.component_handlers.helm_wrapper.utils import trim_release_name
+from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name
 from kpops.component_handlers.kafka_connect.model import (
     KafkaConnectorConfig,
     KafkaConnectorType,
@@ -104,8 +104,7 @@ def helm(self) -> Helm:
     @property
     def _resetter_release_name(self) -> str:
         suffix = "-clean"
-        clean_up_release_name = self.full_name + suffix
-        return trim_release_name(clean_up_release_name, suffix)
+        return create_helm_release_name(self.full_name + suffix, suffix)
 
     @property
     def _resetter_helm_chart(self) -> str:
@@ -244,7 +243,7 @@ def _get_kafka_connect_resetter_values(
                     **kwargs,
                 ),
                 connector_type=self._connector_type.value,
-                name_override=self.full_name,
+                name_override=self.full_name + "-clean",
             ).model_dump(),
             **self.resetter_values,
         }

tests/component_handlers/helm_wrapper/test_utils.py

@@ -1,29 +1,33 @@
-from kpops.component_handlers.helm_wrapper.utils import trim_release_name
+from kpops.component_handlers.helm_wrapper.utils import (
+    create_helm_release_name,
+)
 
 
-def test_trim_release_name_with_suffix():
-    name = trim_release_name(
-        "example-component-name-too-long-fake-fakefakefakefakefake-clean",
-        suffix="-clean",
-    )
-    assert name == "example-component-name-too-long-fake-fakefakef-clean"
-    assert len(name) == 52
+def test_helm_release_name_for_long_names():
+    long_release_name = "example-component-name-too-long-fake-fakefakefakefakefake"
 
+    actual_release_name = create_helm_release_name(long_release_name)
 
-def test_trim_release_name_without_suffix():
-    name = trim_release_name(
-        "example-component-name-too-long-fake-fakefakefakefakefake"
-    )
-    assert name == "example-component-name-too-long-fake-fakefakefakefak"
-    assert len(name) == 52
+    expected_helm_release_name = "example-component-name-too-long-fake-fakefakef-0a7fc"
+    assert expected_helm_release_name == actual_release_name
+    assert len(expected_helm_release_name) == 52
 
 
-def test_no_trim_release_name():
-    assert (
-        trim_release_name("normal-name-with-no-need-of-trim-clean", suffix="-clean")
-        == "normal-name-with-no-need-of-trim-clean"
-    )
-    assert (
-        trim_release_name("normal-name-with-no-need-of-trim")
-        == "normal-name-with-no-need-of-trim"
+def test_helm_release_name_for_install_and_clean_must_be_different():
+    long_release_name = "example-component-name-too-long-fake-fakefakefakefakefake"
+
+    helm_clean_release_name = create_helm_release_name(long_release_name, "-clean")
+    expected_helm_release_name = (
+        "example-component-name-too-long-fake-fakefakef-0a7fc-clean"
     )
+
+    assert expected_helm_release_name != helm_clean_release_name
+
+
+def test_helm_release_name_for_short_names():
+    short_release_name = "example-component-name"
+
+    actual_helm_release_name = create_helm_release_name(short_release_name)
+
+    assert actual_helm_release_name == short_release_name
+    assert len(actual_helm_release_name) < 53

tests/components/test_kafka_app.py

@@ -10,6 +10,7 @@
     HelmRepoConfig,
     HelmUpgradeInstallFlags,
 )
+from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name
 from kpops.components.base_components import KafkaApp
 from kpops.config import KpopsConfig
 
@@ -92,7 +93,7 @@ def test_should_deploy_kafka_app(
 
         print_helm_diff.assert_called_once()
         helm_upgrade_install.assert_called_once_with(
-            "${pipeline_name}-example-name",
+            create_helm_release_name("${pipeline_name}-example-name"),
             "test/test-chart",
             True,
             "test-namespace",

tests/components/test_kafka_connector.py

@@ -14,7 +14,8 @@
 DEFAULTS_PATH = Path(__file__).parent / "resources"
 CONNECTOR_NAME = "test-connector-with-long-name-0123456789abcdefghijklmnop"
 CONNECTOR_FULL_NAME = "${pipeline_name}-" + CONNECTOR_NAME
-CONNECTOR_CLEAN_FULL_NAME = "${pipeline_name}-test-connector-with-long-name-clean"
+CONNECTOR_CLEAN_FULL_NAME = CONNECTOR_FULL_NAME + "-clean"
+CONNECTOR_CLEAN_RELEASE_NAME = "${pipeline_name}-test-connector-with-lon-449ec-clean"
 CONNECTOR_CLASS = "com.bakdata.connect.TestConnector"
 
 
@@ -111,3 +112,19 @@ def test_connector_config_name_override(
                 app={"connector.class": CONNECTOR_CLASS, "name": ""},  # type: ignore[reportGeneralTypeIssues]
                 namespace="test-namespace",
             )
+
+    def test_resetter_release_name(
+        self,
+        config: KpopsConfig,
+        handlers: ComponentHandlers,
+        connector_config: KafkaConnectorConfig,
+    ):
+        connector = KafkaConnector(
+            name=CONNECTOR_NAME,
+            config=config,
+            handlers=handlers,
+            app=connector_config,
+            namespace="test-namespace",
+        )
+        assert connector.app.name == CONNECTOR_FULL_NAME
+        assert connector._resetter_release_name == CONNECTOR_CLEAN_RELEASE_NAME

tests/components/test_kafka_sink_connector.py

@@ -28,6 +28,7 @@
 from kpops.utils.colorify import magentaify
 from tests.components.test_kafka_connector import (
     CONNECTOR_CLEAN_FULL_NAME,
+    CONNECTOR_CLEAN_RELEASE_NAME,
     CONNECTOR_FULL_NAME,
     CONNECTOR_NAME,
     TestKafkaConnector,
@@ -211,11 +212,11 @@ def test_reset_when_dry_run_is_false(
             ),
             mocker.call.helm.uninstall(
                 namespace="test-namespace",
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 dry_run=dry_run,
             ),
             mocker.call.helm.upgrade_install(
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 namespace="test-namespace",
                 chart="bakdata-kafka-connect-resetter/kafka-connect-resetter",
                 dry_run=dry_run,
@@ -231,12 +232,12 @@ def test_reset_when_dry_run_is_false(
                         "connector": CONNECTOR_FULL_NAME,
                         "deleteConsumerGroup": False,
                     },
-                    "nameOverride": CONNECTOR_FULL_NAME,
+                    "nameOverride": CONNECTOR_CLEAN_FULL_NAME,
                 },
             ),
             mocker.call.helm.uninstall(
                 namespace="test-namespace",
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 dry_run=dry_run,
             ),
         ]
@@ -301,11 +302,11 @@ def test_clean_when_dry_run_is_false(
             ),
             mocker.call.helm.uninstall(
                 namespace="test-namespace",
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 dry_run=dry_run,
             ),
             mocker.call.helm.upgrade_install(
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 namespace="test-namespace",
                 chart="bakdata-kafka-connect-resetter/kafka-connect-resetter",
                 dry_run=dry_run,
@@ -321,12 +322,12 @@ def test_clean_when_dry_run_is_false(
                         "connector": CONNECTOR_FULL_NAME,
                         "deleteConsumerGroup": True,
                     },
-                    "nameOverride": CONNECTOR_FULL_NAME,
+                    "nameOverride": CONNECTOR_CLEAN_FULL_NAME,
                 },
             ),
             mocker.call.helm.uninstall(
                 namespace="test-namespace",
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 dry_run=dry_run,
             ),
         ]
@@ -395,11 +396,11 @@ def test_clean_without_to_when_dry_run_is_false(
             ),
             mocker.call.helm.uninstall(
                 namespace="test-namespace",
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 dry_run=dry_run,
             ),
             mocker.call.helm.upgrade_install(
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 namespace="test-namespace",
                 chart="bakdata-kafka-connect-resetter/kafka-connect-resetter",
                 dry_run=dry_run,
@@ -415,12 +416,12 @@ def test_clean_without_to_when_dry_run_is_false(
                         "connector": CONNECTOR_FULL_NAME,
                         "deleteConsumerGroup": True,
                     },
-                    "nameOverride": CONNECTOR_FULL_NAME,
+                    "nameOverride": CONNECTOR_CLEAN_FULL_NAME,
                 },
             ),
             mocker.call.helm.uninstall(
                 namespace="test-namespace",
-                release_name=CONNECTOR_CLEAN_FULL_NAME,
+                release_name=CONNECTOR_CLEAN_RELEASE_NAME,
                 dry_run=dry_run,
             ),
         ]