From eb76f341f37bcf043ac68ae0fbb5eb88d66cfc1f Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 11 Jan 2024 14:00:44 +0200 Subject: [PATCH 1/4] refactor: rename sub vars WIP --- .../resources/variables/config_env_vars.env | 4 +-- .../resources/variables/config_env_vars.md | 4 +-- docs/docs/schema/config.json | 8 ++--- docs/docs/schema/defaults.json | 18 +++++----- docs/docs/schema/pipeline.json | 10 +++--- .../base_components/pipeline_component.py | 4 +-- kpops/config.py | 4 +-- kpops/pipeline.py | 11 +++--- tests/cli/test_kpops_config.py | 4 +-- tests/compiler/test_pipeline_name.py | 36 +++++++++---------- 10 files changed, 52 insertions(+), 51 deletions(-) diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env index cc1f68943..42d2dead8 100644 --- a/docs/docs/resources/variables/config_env_vars.env +++ b/docs/docs/resources/variables/config_env_vars.env @@ -25,10 +25,10 @@ KPOPS_KAFKA_BROKERS # No default value, required KPOPS_DEFAULTS_FILENAME_PREFIX=defaults # topic_name_config.default_output_topic_name # Configures the value for the variable ${output_topic_name} -KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME=${pipeline_name}-${component.name} +KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME=${pipeline.name}-${component.name} # topic_name_config.default_error_topic_name # Configures the value for the variable ${error_topic_name} -KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME=${pipeline_name}-${component.name}-error +KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME=${pipeline.name}-${component.name}-error # schema_registry.enabled # Whether the Schema Registry handler should be initialized. KPOPS_SCHEMA_REGISTRY__ENABLED=False diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index fd635278a..ef0a7726f 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -7,8 +7,8 @@ These variables are a lower priority alternative to the settings in `config.yaml |KPOPS_PIPELINE_BASE_DIR |. |False |Base directory to the pipelines (default is current working directory) |pipeline_base_dir | |KPOPS_KAFKA_BROKERS | |True |The comma separated Kafka brokers address. |kafka_brokers | |KPOPS_DEFAULTS_FILENAME_PREFIX |defaults |False |The name of the defaults file and the prefix of the defaults environment file. |defaults_filename_prefix | -|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline_name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| -|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline_name}-${component.name}-error|False |Configures the value for the variable ${error_topic_name} |topic_name_config.default_error_topic_name | +|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| +|KPOPS_TOPIC_NAME_CONFIG__DEFAULT_ERROR_TOPIC_NAME |${pipeline.name}-${component.name}-error|False |Configures the value for the variable ${error_topic_name} |topic_name_config.default_error_topic_name | |KPOPS_SCHEMA_REGISTRY__ENABLED |False |False |Whether the Schema Registry handler should be initialized. |schema_registry.enabled | |KPOPS_SCHEMA_REGISTRY__URL |http://localhost:8081/ |False |Address of the Schema Registry. |schema_registry.url | |KPOPS_KAFKA_REST__URL |http://localhost:8082/ |False |Address of the Kafka REST Proxy. |kafka_rest.url | diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index c4ed0b1d4..98056fca0 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -119,13 +119,13 @@ "description": "Configure the topic name variables you can use in the pipeline definition.", "properties": { "default_error_topic_name": { - "default": "${pipeline_name}-${component.name}-error", + "default": "${pipeline.name}-${component.name}-error", "description": "Configures the value for the variable ${error_topic_name}", "title": "Default Error Topic Name", "type": "string" }, "default_output_topic_name": { - "default": "${pipeline_name}-${component.name}", + "default": "${pipeline.name}-${component.name}", "description": "Configures the value for the variable ${output_topic_name}", "title": "Default Output Topic Name", "type": "string" @@ -266,8 +266,8 @@ } ], "default": { - "default_error_topic_name": "${pipeline_name}-${component.name}-error", - "default_output_topic_name": "${pipeline_name}-${component.name}" + "default_error_topic_name": "${pipeline.name}-${component.name}-error", + "default_output_topic_name": "${pipeline.name}-${component.name}" }, "description": "Configure the topic name variables you can use in the pipeline definition." } diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json index 137b547e9..d81314997 100644 --- a/docs/docs/schema/defaults.json +++ b/docs/docs/schema/defaults.json @@ -95,7 +95,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -253,7 +253,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -379,7 +379,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -496,7 +496,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -613,7 +613,7 @@ "title": "Offset Topic" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -746,7 +746,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -811,7 +811,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -864,7 +864,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -1113,7 +1113,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json index da2f31fcb..f6bd2eeff 100644 --- a/docs/docs/schema/pipeline.json +++ b/docs/docs/schema/pipeline.json @@ -95,7 +95,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -271,7 +271,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -388,7 +388,7 @@ "title": "Offset Topic" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -494,7 +494,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -743,7 +743,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" diff --git a/kpops/components/base_components/pipeline_component.py b/kpops/components/base_components/pipeline_component.py index b53e03d43..e37e9dcc5 100644 --- a/kpops/components/base_components/pipeline_component.py +++ b/kpops/components/base_components/pipeline_component.py @@ -27,7 +27,7 @@ class PipelineComponent(BaseDefaultsComponent, ABC): :param name: Component name :param prefix: Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string., - defaults to "${pipeline_name}-" + defaults to "${pipeline.name}-" :param from_: Topic(s) and/or components from which the component will read input, defaults to None :param to: Topic(s) into which the component will write output, @@ -36,7 +36,7 @@ class PipelineComponent(BaseDefaultsComponent, ABC): name: str = Field(default=..., description=describe_attr("name", __doc__)) prefix: str = Field( - default="${pipeline_name}-", + default="${pipeline.name}-", description=describe_attr("prefix", __doc__), ) from_: FromSection | None = Field( diff --git a/kpops/config.py b/kpops/config.py index f71444a43..9bb57e104 100644 --- a/kpops/config.py +++ b/kpops/config.py @@ -21,11 +21,11 @@ class TopicNameConfig(BaseSettings): """Configure the topic name variables you can use in the pipeline definition.""" default_output_topic_name: str = Field( - default="${pipeline_name}-${component.name}", + default="${pipeline.name}-${component.name}", description="Configures the value for the variable ${output_topic_name}", ) default_error_topic_name: str = Field( - default="${pipeline_name}-${component.name}-error", + default="${pipeline.name}-${component.name}-error", description="Configures the value for the variable ${error_topic_name}", ) diff --git a/kpops/pipeline.py b/kpops/pipeline.py index 45a39c232..26629686a 100644 --- a/kpops/pipeline.py +++ b/kpops/pipeline.py @@ -266,6 +266,7 @@ def substitute_in_component(self, component_as_dict: dict) -> dict: ) substitution = generate_substitution( config.model_dump(mode="json"), + "config", existing_substitution=component_substitution, separator=".", ) @@ -295,9 +296,9 @@ def set_pipeline_name_env_vars(base_dir: Path, path: Path) -> None: For example, for a given path ./data/v1/dev/pipeline.yaml the pipeline_name would be set to data-v1-dev. Then the sub environment variables are set: - pipeline_name_0 = data - pipeline_name_1 = v1 - pipeline_name_2 = dev + pipeline.name_0 = data + pipeline.name_1 = v1 + pipeline.name_2 = dev :param base_dir: Base directory to the pipeline files :param path: Path to pipeline.yaml file @@ -307,9 +308,9 @@ def set_pipeline_name_env_vars(base_dir: Path, path: Path) -> None: msg = "The pipeline-base-dir should not equal the pipeline-path" raise ValueError(msg) pipeline_name = "-".join(path_without_file) - ENV["pipeline_name"] = pipeline_name + ENV["pipeline.name"] = pipeline_name for level, parent in enumerate(path_without_file): - ENV[f"pipeline_name_{level}"] = parent + ENV[f"pipeline.name_{level}"] = parent @staticmethod def set_environment_name(environment: str | None) -> None: diff --git a/tests/cli/test_kpops_config.py b/tests/cli/test_kpops_config.py index e52b2345a..5c9655ca3 100644 --- a/tests/cli/test_kpops_config.py +++ b/tests/cli/test_kpops_config.py @@ -18,11 +18,11 @@ def test_kpops_config_with_default_values(): assert default_config.defaults_filename_prefix == "defaults" assert ( default_config.topic_name_config.default_output_topic_name - == "${pipeline_name}-${component.name}" + == "${pipeline.name}-${component.name}" ) assert ( default_config.topic_name_config.default_error_topic_name - == "${pipeline_name}-${component.name}-error" + == "${pipeline.name}-${component.name}-error" ) assert default_config.schema_registry.enabled is False assert default_config.schema_registry.url == AnyHttpUrl("http://localhost:8081") diff --git a/tests/compiler/test_pipeline_name.py b/tests/compiler/test_pipeline_name.py index cca9fe88c..99a228cfe 100644 --- a/tests/compiler/test_pipeline_name.py +++ b/tests/compiler/test_pipeline_name.py @@ -13,12 +13,12 @@ def test_should_set_pipeline_name_with_default_base_dir(): PipelineGenerator.set_pipeline_name_env_vars(DEFAULT_BASE_DIR, PIPELINE_PATH) - assert ENV["pipeline_name"] == "some-random-path-for-testing" - assert ENV["pipeline_name_0"] == "some" - assert ENV["pipeline_name_1"] == "random" - assert ENV["pipeline_name_2"] == "path" - assert ENV["pipeline_name_3"] == "for" - assert ENV["pipeline_name_4"] == "testing" + assert ENV["pipeline.name"] == "some-random-path-for-testing" + assert ENV["pipeline.name_0"] == "some" + assert ENV["pipeline.name_1"] == "random" + assert ENV["pipeline.name_2"] == "path" + assert ENV["pipeline.name_3"] == "for" + assert ENV["pipeline.name_4"] == "testing" def test_should_set_pipeline_name_with_specific_relative_base_dir(): @@ -26,9 +26,9 @@ def test_should_set_pipeline_name_with_specific_relative_base_dir(): Path("./some/random/path"), PIPELINE_PATH ) - assert ENV["pipeline_name"] == "for-testing" - assert ENV["pipeline_name_0"] == "for" - assert ENV["pipeline_name_1"] == "testing" + assert ENV["pipeline.name"] == "for-testing" + assert ENV["pipeline.name_0"] == "for" + assert ENV["pipeline.name_1"] == "testing" def test_should_set_pipeline_name_with_specific_absolute_base_dir(): @@ -36,20 +36,20 @@ def test_should_set_pipeline_name_with_specific_absolute_base_dir(): Path("some/random/path"), PIPELINE_PATH ) - assert ENV["pipeline_name"] == "for-testing" - assert ENV["pipeline_name_0"] == "for" - assert ENV["pipeline_name_1"] == "testing" + assert ENV["pipeline.name"] == "for-testing" + assert ENV["pipeline.name_0"] == "for" + assert ENV["pipeline.name_1"] == "testing" def test_should_set_pipeline_name_with_absolute_base_dir(): PipelineGenerator.set_pipeline_name_env_vars(Path.cwd(), PIPELINE_PATH) - assert ENV["pipeline_name"] == "some-random-path-for-testing" - assert ENV["pipeline_name_0"] == "some" - assert ENV["pipeline_name_1"] == "random" - assert ENV["pipeline_name_2"] == "path" - assert ENV["pipeline_name_3"] == "for" - assert ENV["pipeline_name_4"] == "testing" + assert ENV["pipeline.name"] == "some-random-path-for-testing" + assert ENV["pipeline.name_0"] == "some" + assert ENV["pipeline.name_1"] == "random" + assert ENV["pipeline.name_2"] == "path" + assert ENV["pipeline.name_3"] == "for" + assert ENV["pipeline.name_4"] == "testing" def test_should_not_set_pipeline_name_with_the_same_base_dir(): From 4ba53d5573211916c0fb1f6a4c1ced7ac0024647 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 11 Jan 2024 14:36:02 +0200 Subject: [PATCH 2/4] chore: fix all sub var occurrences --- .../pipeline-components/helm-app.yaml | 16 +-- .../pipeline-components/kafka-app.yaml | 18 ++-- .../pipeline-components/kafka-connector.yaml | 16 +-- .../kafka-sink-connector.yaml | 16 +-- .../kafka-source-connector.yaml | 8 +- .../pipeline-components/kubernetes-app.yaml | 16 +-- .../pipeline-components/pipeline.yaml | 102 +++++++++--------- .../pipeline-components/producer-app.yaml | 10 +- .../sections/app-kafka-app.yaml | 2 +- .../sections/app-producer-app.yaml | 2 +- .../sections/app-streams-app.yaml | 2 +- .../pipeline-components/sections/from_.yaml | 8 +- .../pipeline-components/sections/prefix.yaml | 2 +- .../pipeline-components/sections/to.yaml | 6 +- .../pipeline-components/streams-app.yaml | 18 ++-- .../resources/pipeline-config/config.yaml | 4 +- .../pipeline-defaults/defaults-kafka-app.yaml | 2 +- .../defaults-kafka-connector.yaml | 16 +-- .../defaults-kubernetes-app.yaml | 16 +-- .../defaults-producer-app.yaml | 2 +- .../defaults-streams-app.yaml | 2 +- .../resources/pipeline-defaults/defaults.yaml | 38 +++---- .../core-concepts/variables/substitution.md | 2 +- .../bakdata/atm-fraud-detection/config.yaml | 4 +- .../bakdata/atm-fraud-detection/defaults.yaml | 6 +- .../bakdata/atm-fraud-detection/pipeline.yaml | 2 +- .../snapshots/snap_test_schema_generation.py | 10 +- tests/components/test_helm_app.py | 14 +-- tests/components/test_kafka_app.py | 4 +- tests/components/test_kafka_connector.py | 4 +- tests/components/test_kubernetes_app.py | 2 +- tests/components/test_producer_app.py | 2 +- tests/components/test_streams_app.py | 2 +- tests/pipeline/resources/defaults.yaml | 4 +- .../no-topics-defaults/defaults.yaml | 6 +- .../defaults.yaml | 4 +- .../pipeline-with-env-defaults/defaults.yaml | 4 +- .../pipeline-with-short-topics/defaults.yaml | 2 +- .../read-from-component/pipeline.yaml | 2 +- .../temp-trim-release-name/defaults.yaml | 2 +- 40 files changed, 199 insertions(+), 199 deletions(-) diff --git a/docs/docs/resources/pipeline-components/helm-app.yaml b/docs/docs/resources/pipeline-components/helm-app.yaml index 8f0a59c86..1bd2ce3c0 100644 --- a/docs/docs/resources/pipeline-components/helm-app.yaml +++ b/docs/docs/resources/pipeline-components/helm-app.yaml @@ -3,16 +3,16 @@ name: helm-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -28,11 +28,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-components/kafka-app.yaml b/docs/docs/resources/pipeline-components/kafka-app.yaml index cdc49ef28..8581c15a6 100644 --- a/docs/docs/resources/pipeline-components/kafka-app.yaml +++ b/docs/docs/resources/pipeline-components/kafka-app.yaml @@ -4,16 +4,16 @@ name: kafka-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -29,11 +29,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -49,7 +49,7 @@ # add the key-value pairs they need. app: # required streams: # required - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} nameOverride: override-with-this-name # kafka-app-specific imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app diff --git a/docs/docs/resources/pipeline-components/kafka-connector.yaml b/docs/docs/resources/pipeline-components/kafka-connector.yaml index d44aa7bce..ca6cfc6eb 100644 --- a/docs/docs/resources/pipeline-components/kafka-connector.yaml +++ b/docs/docs/resources/pipeline-components/kafka-connector.yaml @@ -2,16 +2,16 @@ name: kafka-connector # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -27,11 +27,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml b/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml index 017511e5b..06d14ffe1 100644 --- a/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml +++ b/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml @@ -3,16 +3,16 @@ name: kafka-sink-connector # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -28,11 +28,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-components/kafka-source-connector.yaml b/docs/docs/resources/pipeline-components/kafka-source-connector.yaml index d4cbcb24c..e38497b65 100644 --- a/docs/docs/resources/pipeline-components/kafka-source-connector.yaml +++ b/docs/docs/resources/pipeline-components/kafka-source-connector.yaml @@ -3,17 +3,17 @@ name: kafka-source-connector # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- # The source connector has no `from` section # from: # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-components/kubernetes-app.yaml b/docs/docs/resources/pipeline-components/kubernetes-app.yaml index 5170768c2..66ed21bb2 100644 --- a/docs/docs/resources/pipeline-components/kubernetes-app.yaml +++ b/docs/docs/resources/pipeline-components/kubernetes-app.yaml @@ -3,16 +3,16 @@ name: kubernetes-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -28,11 +28,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-components/pipeline.yaml b/docs/docs/resources/pipeline-components/pipeline.yaml index 1c6350fbc..a1acf6f8c 100644 --- a/docs/docs/resources/pipeline-components/pipeline.yaml +++ b/docs/docs/resources/pipeline-components/pipeline.yaml @@ -3,16 +3,16 @@ name: helm-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -28,11 +28,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -67,16 +67,16 @@ name: kafka-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -92,11 +92,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -112,7 +112,7 @@ # add the key-value pairs they need. app: # required streams: # required - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} nameOverride: override-with-this-name # kafka-app-specific imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app @@ -132,16 +132,16 @@ name: kafka-sink-connector # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -157,11 +157,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -197,17 +197,17 @@ name: kafka-source-connector # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- # The source connector has no `from` section # from: # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -246,16 +246,16 @@ name: kubernetes-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -271,11 +271,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -301,17 +301,17 @@ name: producer-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- # from: # While the producer-app does inherit from kafka-app, it does not need a # `from` section, hence it does not support it. # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -327,7 +327,7 @@ # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app app: # required streams: # required, producer-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} outputTopic: output_topic extraOutputTopics: @@ -351,16 +351,16 @@ name: streams-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -376,11 +376,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -398,7 +398,7 @@ app: # required # Streams Bootstrap streams section streams: # required, streams-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} inputTopics: - topic1 diff --git a/docs/docs/resources/pipeline-components/producer-app.yaml b/docs/docs/resources/pipeline-components/producer-app.yaml index 5be3551d8..064e83c67 100644 --- a/docs/docs/resources/pipeline-components/producer-app.yaml +++ b/docs/docs/resources/pipeline-components/producer-app.yaml @@ -6,17 +6,17 @@ name: producer-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- # from: # While the producer-app does inherit from kafka-app, it does not need a # `from` section, hence it does not support it. # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -32,7 +32,7 @@ # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app app: # required streams: # required, producer-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} outputTopic: output_topic extraOutputTopics: diff --git a/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml b/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml index 73b70c59e..11dbf7349 100644 --- a/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml +++ b/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml @@ -2,7 +2,7 @@ # add the key-value pairs they need. app: # required streams: # required - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} nameOverride: override-with-this-name # kafka-app-specific imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app diff --git a/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml b/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml index 0cbe04ded..33774aacd 100644 --- a/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml +++ b/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml @@ -2,7 +2,7 @@ # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app app: # required streams: # required, producer-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} outputTopic: output_topic extraOutputTopics: diff --git a/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml b/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml index 1c5f0849f..9054c57cf 100644 --- a/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml +++ b/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml @@ -4,7 +4,7 @@ app: # required # Streams Bootstrap streams section streams: # required, streams-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} inputTopics: - topic1 diff --git a/docs/docs/resources/pipeline-components/sections/from_.yaml b/docs/docs/resources/pipeline-components/sections/from_.yaml index 3f7f0dd22..777d10d0e 100644 --- a/docs/docs/resources/pipeline-components/sections/from_.yaml +++ b/docs/docs/resources/pipeline-components/sections/from_.yaml @@ -1,12 +1,12 @@ from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component diff --git a/docs/docs/resources/pipeline-components/sections/prefix.yaml b/docs/docs/resources/pipeline-components/sections/prefix.yaml index 91fbda223..b4d03f519 100644 --- a/docs/docs/resources/pipeline-components/sections/prefix.yaml +++ b/docs/docs/resources/pipeline-components/sections/prefix.yaml @@ -1,3 +1,3 @@ # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- diff --git a/docs/docs/resources/pipeline-components/sections/to.yaml b/docs/docs/resources/pipeline-components/sections/to.yaml index dd81be9ef..7ebaf60df 100644 --- a/docs/docs/resources/pipeline-components/sections/to.yaml +++ b/docs/docs/resources/pipeline-components/sections/to.yaml @@ -1,11 +1,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-components/streams-app.yaml b/docs/docs/resources/pipeline-components/streams-app.yaml index f77edf80c..d0408b190 100644 --- a/docs/docs/resources/pipeline-components/streams-app.yaml +++ b/docs/docs/resources/pipeline-components/streams-app.yaml @@ -4,16 +4,16 @@ name: streams-app # required # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -29,11 +29,11 @@ # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -51,7 +51,7 @@ app: # required # Streams Bootstrap streams section streams: # required, streams-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} inputTopics: - topic1 diff --git a/docs/docs/resources/pipeline-config/config.yaml b/docs/docs/resources/pipeline-config/config.yaml index 3b08c0708..275382d46 100644 --- a/docs/docs/resources/pipeline-config/config.yaml +++ b/docs/docs/resources/pipeline-config/config.yaml @@ -16,9 +16,9 @@ defaults_filename_prefix: defaults # Configures topic names. topic_name_config: # Configures the value for the variable ${output_topic_name} - default_output_topic_name: ${pipeline_name}-${component_name} + default_output_topic_name: ${pipeline.name}-${component_name} # Configures the value for the variable ${error_topic_name} - default_error_topic_name: ${pipeline_name}-${component_name}-error + default_error_topic_name: ${pipeline.name}-${component_name}-error # Address of the Schema Registry schema_registry_url: "http://localhost:8081" # Address of the Kafka REST Proxy. diff --git a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml index bd6c9e2d9..b2dd5f928 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml @@ -7,7 +7,7 @@ kafka-app: # add the key-value pairs they need. app: # required streams: # required - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} nameOverride: override-with-this-name # kafka-app-specific imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app diff --git a/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml b/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml index 8aa5e8ac2..489bf8bb1 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kafka-connector.yaml @@ -5,16 +5,16 @@ kafka-connector: # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -30,11 +30,11 @@ kafka-connector: # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml index 5dd85e9ce..0780de384 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-kubernetes-app.yaml @@ -5,16 +5,16 @@ kubernetes-app: # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -30,11 +30,11 @@ kubernetes-app: # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use diff --git a/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml index bfa5521c4..5d096a917 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml @@ -10,7 +10,7 @@ producer-app: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app app: # required streams: # required, producer-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} outputTopic: output_topic extraOutputTopics: diff --git a/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml index ae1adab98..9d0907a82 100644 --- a/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml @@ -9,7 +9,7 @@ streams-app: app: # required # Streams Bootstrap streams section streams: # required, streams-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} inputTopics: - topic1 diff --git a/docs/docs/resources/pipeline-defaults/defaults.yaml b/docs/docs/resources/pipeline-defaults/defaults.yaml index 58b22d3f3..78d351d01 100644 --- a/docs/docs/resources/pipeline-defaults/defaults.yaml +++ b/docs/docs/resources/pipeline-defaults/defaults.yaml @@ -28,7 +28,7 @@ kafka-app: # add the key-value pairs they need. app: # required streams: # required - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} nameOverride: override-with-this-name # kafka-app-specific imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app @@ -40,16 +40,16 @@ kafka-app: kafka-connector: # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -65,11 +65,11 @@ kafka-connector: # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -121,16 +121,16 @@ kafka-source-connector: kubernetes-app: # Pipeline prefix that will prefix every component name. If you wish to not # have any prefix you can specify an empty string. - prefix: ${pipeline_name}- + prefix: ${pipeline.name}- from: # Must not be null topics: # read from topic - ${pipeline_name}-input-topic: + ${pipeline.name}-input-topic: type: input # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra - ${pipeline_name}-input-pattern-topic: + ${pipeline.name}-input-pattern-topic: type: pattern # Implied to be an input pattern if `role` is undefined - ${pipeline_name}-extra-pattern-topic: + ${pipeline.name}-extra-pattern-topic: type: pattern # Implied to be an extra pattern if `role` is defined role: some-role components: # read from specific component @@ -146,11 +146,11 @@ kubernetes-app: # Topic(s) into which the component will write output to: topics: - ${pipeline_name}-output-topic: + ${pipeline.name}-output-topic: type: output # Implied when role is NOT specified - ${pipeline_name}-extra-topic: + ${pipeline.name}-extra-topic: role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined - ${pipeline_name}-error-topic: + ${pipeline.name}-error-topic: type: error # Currently KPOps supports Avro and JSON schemas. key_schema: key-schema # must implement SchemaProvider to use @@ -180,7 +180,7 @@ producer-app: # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app app: # required streams: # required, producer-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} outputTopic: output_topic extraOutputTopics: @@ -198,7 +198,7 @@ streams-app: app: # required # Streams Bootstrap streams section streams: # required, streams-app-specific - brokers: ${kafka_brokers} # required + brokers: ${config.kafka_brokers} # required schemaRegistryUrl: ${schema_registry_url} inputTopics: - topic1 diff --git a/docs/docs/user/core-concepts/variables/substitution.md b/docs/docs/user/core-concepts/variables/substitution.md index 71782180d..b1bfa97e3 100644 --- a/docs/docs/user/core-concepts/variables/substitution.md +++ b/docs/docs/user/core-concepts/variables/substitution.md @@ -41,7 +41,7 @@ Environment variables such as `$PATH` can be used in the pipeline definition and These are special variables that refer to the name and path of a pipeline. -- `${pipeline_name}`: Concatenated path of the parent directory where pipeline.yaml is defined in. +- `${pipeline.name}`: Concatenated path of the parent directory where pipeline.yaml is defined in. For instance, `./data/pipelines/v1/pipeline.yaml`, here the value for the variable would be `data-pipelines-v1`. - `${pipeline_name_}`: Similar to the previous variable, each `` contains a part of the path to the `pipeline.yaml` file. diff --git a/examples/bakdata/atm-fraud-detection/config.yaml b/examples/bakdata/atm-fraud-detection/config.yaml index c3195147b..c20493eb7 100644 --- a/examples/bakdata/atm-fraud-detection/config.yaml +++ b/examples/bakdata/atm-fraud-detection/config.yaml @@ -1,6 +1,6 @@ topic_name_config: - default_error_topic_name: "${pipeline_name}-${component.name}-dead-letter-topic" - default_output_topic_name: "${pipeline_name}-${component.name}-topic" + default_error_topic_name: "${pipeline.name}-${component.name}-dead-letter-topic" + default_output_topic_name: "${pipeline.name}-${component.name}-topic" kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" diff --git a/examples/bakdata/atm-fraud-detection/defaults.yaml b/examples/bakdata/atm-fraud-detection/defaults.yaml index 2e9079f4f..a5a060378 100644 --- a/examples/bakdata/atm-fraud-detection/defaults.yaml +++ b/examples/bakdata/atm-fraud-detection/defaults.yaml @@ -10,8 +10,8 @@ kafka-connector: kafka-app: app: streams: - brokers: ${kafka_brokers} - schemaRegistryUrl: ${schema_registry.url} + brokers: ${config.kafka_brokers} + schemaRegistryUrl: ${config.schema_registry.url} optimizeLeaveGroupBehavior: false producer-app: @@ -23,7 +23,7 @@ producer-app: streams-app: app: labels: - pipeline: ${pipeline_name} + pipeline: ${pipeline.name} streams: optimizeLeaveGroupBehavior: false to: diff --git a/examples/bakdata/atm-fraud-detection/pipeline.yaml b/examples/bakdata/atm-fraud-detection/pipeline.yaml index 9982aa0a7..d166a21f4 100644 --- a/examples/bakdata/atm-fraud-detection/pipeline.yaml +++ b/examples/bakdata/atm-fraud-detection/pipeline.yaml @@ -83,7 +83,7 @@ app: connector.class: io.confluent.connect.jdbc.JdbcSinkConnector tasks.max: 1 - topics: ${pipeline_name}-account-linker-topic + topics: ${pipeline.name}-account-linker-topic connection.url: jdbc:postgresql://postgresql-dev.${NAMESPACE}.svc.cluster.local:5432/app_db connection.user: app1 connection.password: AppPassword diff --git a/tests/cli/snapshots/snap_test_schema_generation.py b/tests/cli/snapshots/snap_test_schema_generation.py index 4875c610b..f23e77422 100644 --- a/tests/cli/snapshots/snap_test_schema_generation.py +++ b/tests/cli/snapshots/snap_test_schema_generation.py @@ -32,7 +32,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -161,7 +161,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -213,7 +213,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -270,7 +270,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" @@ -323,7 +323,7 @@ "type": "string" }, "prefix": { - "default": "${pipeline_name}-", + "default": "${pipeline.name}-", "description": "Pipeline prefix that will prefix every component name. If you wish to not have any prefix you can specify an empty string.", "title": "Prefix", "type": "string" diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py index e43c9de41..f01f30d10 100644 --- a/tests/components/test_helm_app.py +++ b/tests/components/test_helm_app.py @@ -88,12 +88,12 @@ def test_should_lazy_load_helm_wrapper_and_not_repo_add( helm_app.deploy(False) helm_mock.upgrade_install.assert_called_once_with( - "${pipeline_name}-test-helm-app", + "${pipeline.name}-test-helm-app", "test/test-chart", False, "test-namespace", { - "nameOverride": "${pipeline_name}-test-helm-app", + "nameOverride": "${pipeline.name}-test-helm-app", "foo": "test-value", }, HelmUpgradeInstallFlags(), @@ -136,12 +136,12 @@ def test_should_lazy_load_helm_wrapper_and_call_repo_add_when_implemented( RepoAuthFlags(), ), mocker.call.upgrade_install( - "${pipeline_name}-test-helm-app", + "${pipeline.name}-test-helm-app", "test/test-chart", False, "test-namespace", { - "nameOverride": "${pipeline_name}-test-helm-app", + "nameOverride": "${pipeline.name}-test-helm-app", "foo": "test-value", }, HelmUpgradeInstallFlags(version="3.4.5"), @@ -176,12 +176,12 @@ def helm_chart(self) -> str: helm_mock.add_repo.assert_not_called() helm_mock.upgrade_install.assert_called_once_with( - "${pipeline_name}-test-app-with-local-chart", + "${pipeline.name}-test-app-with-local-chart", "path/to/helm/charts/", False, "test-namespace", { - "nameOverride": "${pipeline_name}-test-app-with-local-chart", + "nameOverride": "${pipeline.name}-test-app-with-local-chart", "foo": "test-value", }, HelmUpgradeInstallFlags(), @@ -212,7 +212,7 @@ def test_should_call_helm_uninstall_when_destroying_helm_app( helm_app.destroy(True) helm_mock.uninstall.assert_called_once_with( - "test-namespace", "${pipeline_name}-test-helm-app", True + "test-namespace", "${pipeline.name}-test-helm-app", True ) log_info_mock.assert_called_once_with(magentaify(stdout)) diff --git a/tests/components/test_kafka_app.py b/tests/components/test_kafka_app.py index d7e8fd5d4..21c9072f8 100644 --- a/tests/components/test_kafka_app.py +++ b/tests/components/test_kafka_app.py @@ -93,12 +93,12 @@ def test_should_deploy_kafka_app( print_helm_diff.assert_called_once() helm_upgrade_install.assert_called_once_with( - create_helm_release_name("${pipeline_name}-example-name"), + create_helm_release_name("${pipeline.name}-example-name"), "test/test-chart", True, "test-namespace", { - "nameOverride": "${pipeline_name}-example-name", + "nameOverride": "${pipeline.name}-example-name", "streams": {"brokers": "fake-broker:9092", "outputTopic": "test"}, }, HelmUpgradeInstallFlags(version="1.2.3"), diff --git a/tests/components/test_kafka_connector.py b/tests/components/test_kafka_connector.py index 6c0e0dcc3..d352a6d8a 100644 --- a/tests/components/test_kafka_connector.py +++ b/tests/components/test_kafka_connector.py @@ -13,9 +13,9 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" CONNECTOR_NAME = "test-connector-with-long-name-0123456789abcdefghijklmnop" -CONNECTOR_FULL_NAME = "${pipeline_name}-" + CONNECTOR_NAME +CONNECTOR_FULL_NAME = "${pipeline.name}-" + CONNECTOR_NAME CONNECTOR_CLEAN_FULL_NAME = CONNECTOR_FULL_NAME + "-clean" -CONNECTOR_CLEAN_RELEASE_NAME = "${pipeline_name}-test-connector-with-lon-449ec-clean" +CONNECTOR_CLEAN_RELEASE_NAME = "${pipeline.name}-test-connector-with-lon-612f3-clean" CONNECTOR_CLASS = "com.bakdata.connect.TestConnector" diff --git a/tests/components/test_kubernetes_app.py b/tests/components/test_kubernetes_app.py index ebc2701fa..c949f9832 100644 --- a/tests/components/test_kubernetes_app.py +++ b/tests/components/test_kubernetes_app.py @@ -15,7 +15,7 @@ ) from kpops.config import KpopsConfig -HELM_RELEASE_NAME = create_helm_release_name("${pipeline_name}-test-kubernetes-app") +HELM_RELEASE_NAME = create_helm_release_name("${pipeline.name}-test-kubernetes-app") DEFAULTS_PATH = Path(__file__).parent / "resources" diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 07e78bb6a..2038c6909 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -18,7 +18,7 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" PRODUCER_APP_NAME = "test-producer-app-with-long-name-0123456789abcdefghijklmnop" -PRODUCER_APP_FULL_NAME = "${pipeline_name}-" + PRODUCER_APP_NAME +PRODUCER_APP_FULL_NAME = "${pipeline.name}-" + PRODUCER_APP_NAME PRODUCER_APP_RELEASE_NAME = create_helm_release_name(PRODUCER_APP_FULL_NAME) PRODUCER_APP_CLEAN_FULL_NAME = PRODUCER_APP_FULL_NAME + "-clean" PRODUCER_APP_CLEAN_RELEASE_NAME = create_helm_release_name( diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index ebd5cf7d9..2de276643 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -22,7 +22,7 @@ DEFAULTS_PATH = Path(__file__).parent / "resources" STREAMS_APP_NAME = "test-streams-app-with-long-name-0123456789abcdefghijklmnop" -STREAMS_APP_FULL_NAME = "${pipeline_name}-" + STREAMS_APP_NAME +STREAMS_APP_FULL_NAME = "${pipeline.name}-" + STREAMS_APP_NAME STREAMS_APP_RELEASE_NAME = create_helm_release_name(STREAMS_APP_FULL_NAME) STREAMS_APP_CLEAN_FULL_NAME = STREAMS_APP_FULL_NAME + "-clean" STREAMS_APP_CLEAN_RELEASE_NAME = create_helm_release_name( diff --git a/tests/pipeline/resources/defaults.yaml b/tests/pipeline/resources/defaults.yaml index b78293627..101e3e175 100644 --- a/tests/pipeline/resources/defaults.yaml +++ b/tests/pipeline/resources/defaults.yaml @@ -5,8 +5,8 @@ kubernetes-app: kafka-app: app: streams: - brokers: "${kafka_brokers}" - schema_registry_url: "${schema_registry.url}" + brokers: "${config.kafka_brokers}" + schema_registry_url: "${config.schema_registry.url}" version: "2.4.2" producer-app: {} # inherits from kafka-app diff --git a/tests/pipeline/resources/no-topics-defaults/defaults.yaml b/tests/pipeline/resources/no-topics-defaults/defaults.yaml index ea3dd7d9e..7820898a3 100644 --- a/tests/pipeline/resources/no-topics-defaults/defaults.yaml +++ b/tests/pipeline/resources/no-topics-defaults/defaults.yaml @@ -1,8 +1,8 @@ kafka-app: app: streams: - brokers: "${kafka_brokers}" - schemaRegistryUrl: "${schema_registry.url}" + brokers: "${config.kafka_brokers}" + schemaRegistryUrl: "${config.schema_registry.url}" producer-app: to: @@ -14,7 +14,7 @@ producer-app: streams-app: app: labels: - pipeline: ${pipeline_name} + pipeline: ${pipeline.name} to: topics: ${error_topic_name}: diff --git a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml index b5954da19..ff053e990 100644 --- a/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml +++ b/tests/pipeline/resources/pipeline-component-should-have-prefix/defaults.yaml @@ -7,5 +7,5 @@ kubernetes-app: kafka-app: app: streams: - brokers: ${kafka_brokers} - schemaRegistryUrl: ${schema_registry.url} + brokers: ${config.kafka_brokers} + schemaRegistryUrl: ${config.schema_registry.url} diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml index f9505c0ab..b8aeb6137 100644 --- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml @@ -4,8 +4,8 @@ kubernetes-app: kafka-app: app: streams: - brokers: "${kafka_brokers}" - schemaRegistryUrl: "${schema_registry.url}" + brokers: "${config.kafka_brokers}" + schemaRegistryUrl: "${config.schema_registry.url}" producer-app: {} # inherits from kafka-app diff --git a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml index 3b9e93eb7..cf3b4831b 100644 --- a/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml +++ b/tests/pipeline/resources/pipeline-with-short-topics/defaults.yaml @@ -5,7 +5,7 @@ kubernetes-app: kafka-app: app: streams: - brokers: "${kafka_brokers}" + brokers: "${config.kafka_brokers}" schema_registry_url: "${schema_registry_url}" version: "2.4.2" diff --git a/tests/pipeline/resources/read-from-component/pipeline.yaml b/tests/pipeline/resources/read-from-component/pipeline.yaml index 902e8edd9..cc6bf72c7 100644 --- a/tests/pipeline/resources/read-from-component/pipeline.yaml +++ b/tests/pipeline/resources/read-from-component/pipeline.yaml @@ -44,7 +44,7 @@ name: consumer3 from: topics: - ${pipeline_name}-producer1: + ${pipeline.name}-producer1: type: input components: producer2: diff --git a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml index 55754eba1..c895105b7 100644 --- a/tests/pipeline/resources/temp-trim-release-name/defaults.yaml +++ b/tests/pipeline/resources/temp-trim-release-name/defaults.yaml @@ -4,7 +4,7 @@ kubernetes-app: kafka-app: app: streams: - brokers: "${kafka_brokers}" + brokers: "${config.kafka_brokers}" schema_registry_url: "${schema_registry_url}" version: "2.4.2" From e971d2a9484ebb634330fd1832a5a561b1ec03a3 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 11 Jan 2024 14:46:10 +0200 Subject: [PATCH 3/4] docs: Document changes --- docs/docs/user/migration-guide/v2-v3.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/docs/user/migration-guide/v2-v3.md b/docs/docs/user/migration-guide/v2-v3.md index 9d94af628..40d2fc495 100644 --- a/docs/docs/user/migration-guide/v2-v3.md +++ b/docs/docs/user/migration-guide/v2-v3.md @@ -188,3 +188,21 @@ If you're using this functionality in your custom components, it needs to be upd """Render final component resources, e.g. Kubernetes manifests.""" return [] # list of manifests ``` + +## [Namespace substitution vars](https://github.com/bakdata/kpops/pull/408) + +The global configuration variables is now namespaced under the config key, such as `${config.kafka_brokers}`, `${config.schema_registry.url}`. Same with pipeline variables, e.g. `${pipeline_name} → ${pipeline.name}`. +This would make it more uniform with the existing `${component.}` variables. + +### pipeline.yaml +```diff + name: kafka-app +- prefix: ${pipeline_name}- ++ prefix: ${pipeline.name}- + app: + streams: +- brokers: ${kafka_brokers} +- schemaRegistryUrl: ${schema_registry.url} ++ brokers: ${config.kafka_brokers} ++ schemaRegistryUrl: ${config.schema_registry.url} +``` From b291ad1141a397f9c9016dd38a99ed04722bd806 Mon Sep 17 00:00:00 2001 From: Ivan Yordanov Date: Thu, 11 Jan 2024 14:57:52 +0200 Subject: [PATCH 4/4] fix: grammar --- docs/docs/user/migration-guide/v2-v3.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/docs/user/migration-guide/v2-v3.md b/docs/docs/user/migration-guide/v2-v3.md index 40d2fc495..c4b42c3fa 100644 --- a/docs/docs/user/migration-guide/v2-v3.md +++ b/docs/docs/user/migration-guide/v2-v3.md @@ -191,10 +191,11 @@ If you're using this functionality in your custom components, it needs to be upd ## [Namespace substitution vars](https://github.com/bakdata/kpops/pull/408) -The global configuration variables is now namespaced under the config key, such as `${config.kafka_brokers}`, `${config.schema_registry.url}`. Same with pipeline variables, e.g. `${pipeline_name} → ${pipeline.name}`. +The global configuration variables are now namespaced under the config key, such as `${config.kafka_brokers}`, `${config.schema_registry.url}`. Same with pipeline variables, e.g. `${pipeline_name} → ${pipeline.name}`. This would make it more uniform with the existing `${component.}` variables. ### pipeline.yaml + ```diff name: kafka-app - prefix: ${pipeline_name}-