tests: Update all substitution variables

bakdata · Nov 9, 2023 · c800eb7 · c800eb7
1 parent 941927d
commit c800eb7
Show file tree

Hide file tree

Showing 16 changed files with 35 additions and 34 deletions.
diff --git a/examples/bakdata/atm-fraud-detection/config.yaml b/examples/bakdata/atm-fraud-detection/config.yaml
@@ -1,8 +1,8 @@
 environment: development
 
 topic_name_config:
-  default_error_topic_name: "${pipeline_name}-${component_name}-dead-letter-topic"
-  default_output_topic_name: "${pipeline_name}-${component_name}-topic"
+  default_error_topic_name: "${pipeline_name}-${component__name}-dead-letter-topic"
+  default_output_topic_name: "${pipeline_name}-${component__name}-topic"
 
 kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
 

diff --git a/examples/bakdata/atm-fraud-detection/defaults.yaml b/examples/bakdata/atm-fraud-detection/defaults.yaml
@@ -11,7 +11,7 @@ kafka-app:
   app:
     streams:
       brokers: ${kafka_brokers}
-      schemaRegistryUrl: ${schema_registry_url}
+      schemaRegistryUrl: ${schema_registry__url}
       optimizeLeaveGroupBehavior: false
 
 producer-app:

diff --git a/kpops/config.py b/kpops/config.py
@@ -19,11 +19,11 @@ class TopicNameConfig(BaseSettings):
     """Configure the topic name variables you can use in the pipeline definition."""
 
     default_output_topic_name: str = Field(
-        default="${pipeline_name}-${component_name}",
+        default="${pipeline_name}-${component__name}",
         description="Configures the value for the variable ${output_topic_name}",
     )
     default_error_topic_name: str = Field(
-        default="${pipeline_name}-${component_name}-error",
+        default="${pipeline_name}-${component__name}-error",
         description="Configures the value for the variable ${error_topic_name}",
     )
 

diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline_generator/pipeline.py
@@ -288,9 +288,10 @@ def substitute_in_component(self, component_as_dict: dict) -> dict:
             component_as_dict,
             "component",
             substitution_hardcoded,
+            separator="__",
         )
         substitution = generate_substitution(
-            config.model_dump(mode="json"), existing_substitution=component_substitution
+            config.model_dump(mode="json"), existing_substitution=component_substitution, separator="__"
         )
 
         return json.loads(

diff --git a/tests/cli/test_kpops_config.py b/tests/cli/test_kpops_config.py
@@ -20,11 +20,11 @@ def test_kpops_config_with_default_values():
     assert default_config.defaults_filename_prefix == "defaults"
     assert (
         default_config.topic_name_config.default_output_topic_name
-        == "${pipeline_name}-${component_name}"
+        == "${pipeline_name}-${component__name}"
     )
     assert (
         default_config.topic_name_config.default_error_topic_name
-        == "${pipeline_name}-${component_name}-error"
+        == "${pipeline_name}-${component__name}-error"
     )
     assert default_config.schema_registry.enabled is False
     assert default_config.schema_registry.url == AnyHttpUrl("http://localhost:8081")

diff --git a/tests/pipeline/resources/component-type-substitution/infinite_pipeline.yaml b/tests/pipeline/resources/component-type-substitution/infinite_pipeline.yaml
@@ -1,6 +1,6 @@
 - type: converter
   app:
     labels:
-      l_1: ${component_app_labels_l_2}
-      l_2: ${component_app_labels_l_1}
-      infinite_nesting: ${component_app_labels}
+      l_1: ${component__app__labels__l_2}
+      l_2: ${component__app__labels__l_1}
+      infinite_nesting: ${component__app__labels}
diff --git a/tests/pipeline/resources/component-type-substitution/pipeline.yaml b/tests/pipeline/resources/component-type-substitution/pipeline.yaml
@@ -1,9 +1,9 @@
 - type: scheduled-producer
   app:
     labels:
-      app_type: "${component_type}"
-      app_name: "${component_name}"
-      app_schedule: "${component_app_schedule}"
+      app_type: "${component__type}"
+      app_name: "${component__name}"
+      app_schedule: "${component__app__schedule}"
     commandLine:
       FAKE_ARG: "fake-arg-value"
     schedule: "30 3/8 * * *"
@@ -20,11 +20,11 @@
   name: "filter-app"
   app:
     labels:
-      app_type: "${component_type}"
-      app_name: "${component_name}"
-      app_resources_requests_memory: "${component_app_resources_requests_memory}"
-      ${component_type}: "${component_app_labels_app_name}-${component_app_labels_app_type}"
-      test_placeholder_in_placeholder: "${component_app_labels_${component_type}}"
+      app_type: "${component__type}"
+      app_name: "${component__name}"
+      app_resources_requests_memory: "${component__app__resources__requests__memory}"
+      ${component__type}: "${component__app__labels__app_name}-${component__app__labels__app_type}"
+      test_placeholder_in_placeholder: "${component__app__labels__${component__type}}"
     commandLine:
       TYPE: "nothing"
     resources:

diff --git a/tests/pipeline/resources/custom-config/config.yaml b/tests/pipeline/resources/custom-config/config.yaml
@@ -1,8 +1,8 @@
 environment: development
 defaults_path: ../no-topics-defaults
 topic_name_config:
-  default_error_topic_name: "${component_name}-dead-letter-topic"
-  default_output_topic_name: "${component_name}-test-topic"
+  default_error_topic_name: "${component__name}-dead-letter-topic"
+  default_output_topic_name: "${component__name}-test-topic"
 kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
 kafka_connect:
   url: "http://localhost:8083"

diff --git a/tests/pipeline/resources/defaults.yaml b/tests/pipeline/resources/defaults.yaml
@@ -1,12 +1,12 @@
 kubernetes-app:
-  name: "${component_type}"
+  name: "${component__type}"
   namespace: example-namespace
 
 kafka-app:
   app:
     streams:
       brokers: "${kafka_brokers}"
-      schema_registry_url: "${schema_registry_url}"
+      schema_registry_url: "${schema_registry__url}"
   version: "2.4.2"
 
 producer-app: {} # inherits from kafka-app

diff --git a/tests/pipeline/resources/kafka-connect-sink-config/config.yaml b/tests/pipeline/resources/kafka-connect-sink-config/config.yaml
@@ -2,8 +2,8 @@ environment: development
 defaults_path: ..
 kafka_brokers: "broker:9092"
 topic_name_config: 
-  default_error_topic_name: ${component_type}-error-topic
-  default_output_topic_name: ${component_type}-output-topic
+  default_error_topic_name: ${component__type}-error-topic
+  default_output_topic_name: ${component__type}-output-topic
 helm_diff_config: 
   enable: false
 kafka_connect:

diff --git a/tests/pipeline/resources/no-topics-defaults/defaults.yaml b/tests/pipeline/resources/no-topics-defaults/defaults.yaml
@@ -2,7 +2,7 @@ kafka-app:
   app:
     streams:
       brokers: "${kafka_brokers}"
-      schemaRegistryUrl: "${schema_registry_url}"
+      schemaRegistryUrl: "${schema_registry__url}"
 
 producer-app:
   to:

diff --git a/tests/pipeline/resources/no-topics-defaults/defaults_development.yaml b/tests/pipeline/resources/no-topics-defaults/defaults_development.yaml
@@ -1,3 +1,3 @@
 kubernetes-app:
-  name: "${component_type}-development"
+  name: "${component__type}-development"
   namespace: development-namespace
diff --git a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml
@@ -8,4 +8,4 @@ kafka-app:
   app:
     streams:
       brokers: ${kafka_brokers}
-      schemaRegistryUrl: ${schema_registry_url}
+      schemaRegistryUrl: ${schema_registry__url}
diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
@@ -1,11 +1,11 @@
 kubernetes-app:
-  name: ${component_type}
+  name: ${component__type}
   namespace: example-namespace
 kafka-app:
   app:
     streams:
       brokers: "${kafka_brokers}"
-      schemaRegistryUrl: "${schema_registry_url}"
+      schemaRegistryUrl: "${schema_registry__url}"
 
 producer-app: {} # inherits from kafka-app
 

diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults_development.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults_development.yaml
@@ -1,3 +1,3 @@
 kubernetes-app:
-  name: ${component_type}-development
+  name: ${component__type}-development
   namespace: development-namespace
diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py
@@ -51,10 +51,10 @@ def inflate(self) -> list[PipelineComponent]:
                         },
                         to=ToSection(
                             topics={
-                                TopicName("${component_type}"): TopicConfig(
+                                TopicName("${component__type}"): TopicConfig(
                                     type=OutputTopicTypes.OUTPUT
                                 ),
-                                TopicName("${component_name}"): TopicConfig(
+                                TopicName("${component__name}"): TopicConfig(
                                     type=None, role="test"
                                 ),
                             }
@@ -68,7 +68,7 @@ def inflate(self) -> list[PipelineComponent]:
                         to=ToSection(  # type: ignore[reportGeneralTypeIssues]
                             topics={
                                 TopicName(
-                                    f"{self.full_name}-" + "${component_name}"
+                                    f"{self.full_name}-" + "${component__name}"
                                 ): TopicConfig(type=OutputTopicTypes.OUTPUT)
                             }
                         ).model_dump(),