From f95afe23e09704f9bbfa5d8c3c59d472b2eb1533 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 20 Dec 2023 15:55:33 +0100 Subject: [PATCH] Refactor pipeline generator & representation (#392) Fixes #381 --- hooks/gen_docs/gen_docs_components.py | 2 +- kpops/cli/main.py | 10 +- .../base_defaults_component.py | 2 +- kpops/{pipeline_generator => }/pipeline.py | 140 +- kpops/pipeline_generator/__init__.py | 0 kpops/utils/pydantic.py | 2 +- kpops/utils/{yaml_loading.py => yaml.py} | 19 + tests/cli/test_pipeline_steps.py | 19 +- tests/compiler/test_pipeline_name.py | 18 +- tests/compiler/test_yaml_loading.py | 2 +- .../test_base_defaults_component.py | 4 +- tests/conftest.py | 2 +- tests/pipeline/snapshots/snap_test_example.py | 612 ++- .../pipeline/snapshots/snap_test_pipeline.py | 4338 ++++++++--------- tests/pipeline/test_pipeline.py | 61 +- 15 files changed, 2593 insertions(+), 2638 deletions(-) rename kpops/{pipeline_generator => }/pipeline.py (78%) delete mode 100644 kpops/pipeline_generator/__init__.py rename kpops/utils/{yaml_loading.py => yaml.py} (83%) diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py index 203294c05..f1acf9973 100644 --- a/hooks/gen_docs/gen_docs_components.py +++ b/hooks/gen_docs/gen_docs_components.py @@ -11,7 +11,7 @@ from kpops.cli.registry import _find_classes from kpops.components import KafkaConnector, PipelineComponent from kpops.utils.colorify import redify, yellowify -from kpops.utils.yaml_loading import load_yaml_file +from kpops.utils.yaml import load_yaml_file PATH_KPOPS_MAIN = ROOT / "kpops/cli/main.py" PATH_CLI_COMMANDS_DOC = ROOT / "docs/docs/user/references/cli-commands.md" diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 285c1154f..5a7c758e4 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -19,9 +19,10 @@ from kpops.component_handlers.topic.handler import TopicHandler from kpops.component_handlers.topic.proxy_wrapper import ProxyWrapper from kpops.config import ENV_PREFIX, KpopsConfig -from kpops.pipeline_generator.pipeline import Pipeline +from kpops.pipeline import Pipeline, PipelineGenerator from kpops.utils.gen_schema import SchemaScope, gen_config_schema, gen_pipeline_schema from kpops.utils.pydantic import YamlConfigSettingsSource +from kpops.utils.yaml import print_yaml if TYPE_CHECKING: from collections.abc import Iterator @@ -144,9 +145,8 @@ def setup_pipeline( registry.find_components("kpops.components") handlers = setup_handlers(components_module, kpops_config) - return Pipeline.load_from_yaml( - pipeline_base_dir, pipeline_path, environment, registry, kpops_config, handlers - ) + parser = PipelineGenerator(kpops_config, registry, handlers) + return parser.load_yaml(pipeline_base_dir, pipeline_path, environment) def setup_handlers( @@ -288,7 +288,7 @@ def generate( ) if not template: - pipeline.print_yaml() + print_yaml(pipeline.to_yaml()) if template: steps_to_apply = get_steps_to_apply(pipeline, steps, filter_type) diff --git a/kpops/components/base_components/base_defaults_component.py b/kpops/components/base_components/base_defaults_component.py index 7ef978705..fff9135da 100644 --- a/kpops/components/base_components/base_defaults_component.py +++ b/kpops/components/base_components/base_defaults_component.py @@ -18,7 +18,7 @@ from kpops.utils.docstring import describe_attr from kpops.utils.environment import ENV from kpops.utils.pydantic import DescConfigModel, to_dash -from kpops.utils.yaml_loading import load_yaml_file +from kpops.utils.yaml import load_yaml_file try: from typing import Self diff --git a/kpops/pipeline_generator/pipeline.py b/kpops/pipeline.py similarity index 78% rename from kpops/pipeline_generator/pipeline.py rename to kpops/pipeline.py index 07ea94e1e..de37576d0 100644 --- a/kpops/pipeline_generator/pipeline.py +++ b/kpops/pipeline.py @@ -4,17 +4,16 @@ import logging from collections import Counter from contextlib import suppress +from dataclasses import dataclass, field from typing import TYPE_CHECKING import yaml -from pydantic import BaseModel, SerializeAsAny -from rich.console import Console -from rich.syntax import Syntax +from pydantic import Field, RootModel, SerializeAsAny from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.utils.dict_ops import generate_substitution, update_nested_pair from kpops.utils.environment import ENV -from kpops.utils.yaml_loading import load_yaml_file, substitute, substitute_nested +from kpops.utils.yaml import load_yaml_file, substitute_nested if TYPE_CHECKING: from collections.abc import Iterator @@ -35,17 +34,19 @@ class ValidationError(Exception): pass -class PipelineComponents(BaseModel): - """Stores the pipeline components.""" +class Pipeline(RootModel): + """Pipeline representation.""" - components: list[SerializeAsAny[PipelineComponent]] = [] + root: list[SerializeAsAny[PipelineComponent]] = Field( + default=[], title="Components" + ) @property def last(self) -> PipelineComponent: - return self.components[-1] + return self.root[-1] def find(self, component_name: str) -> PipelineComponent: - for component in self.components: + for component in self.root: if component_name == component.name: return component msg = f"Component {component_name} not found" @@ -53,19 +54,25 @@ def find(self, component_name: str) -> PipelineComponent: def add(self, component: PipelineComponent) -> None: self._populate_component_name(component) - self.components.append(component) + self.root.append(component) def __bool__(self) -> bool: - return bool(self.components) + return bool(self.root) def __iter__(self) -> Iterator[PipelineComponent]: - return iter(self.components) + return iter(self.root) def __len__(self) -> int: - return len(self.components) + return len(self.root) + + def to_yaml(self) -> str: + return yaml.dump(self.model_dump(mode="json", by_alias=True, exclude_none=True)) + + def validate(self) -> None: + self.validate_unique_names() def validate_unique_names(self) -> None: - step_names = [component.full_name for component in self.components] + step_names = [component.full_name for component in self.root] duplicates = [name for name, count in Counter(step_names).items() if count > 1] if duplicates: msg = f"step names should be unique. duplicate step names: {', '.join(duplicates)}" @@ -97,32 +104,31 @@ def create_env_components_index( return index -class Pipeline: - def __init__( +@dataclass +class PipelineGenerator: + config: KpopsConfig + registry: Registry + handlers: ComponentHandlers + pipeline: Pipeline = field(init=False, default_factory=Pipeline) + + def parse( self, - component_list: list[dict], + components: list[dict], environment_components: list[dict], - registry: Registry, - config: KpopsConfig, - handlers: ComponentHandlers, - ) -> None: - self.components: PipelineComponents = PipelineComponents() - self.handlers = handlers - self.config = config - self.registry = registry + ) -> Pipeline: + """Parse pipeline from sequence of component dictionaries. + + :param components: List of components + :param environment_components: List of environment-specific components + :returns: Initialized pipeline object + """ self.env_components_index = create_env_components_index(environment_components) - self.parse_components(component_list) - self.validate() - - @classmethod - def load_from_yaml( - cls, - base_dir: Path, - path: Path, - environment: str | None, - registry: Registry, - config: KpopsConfig, - handlers: ComponentHandlers, + self.parse_components(components) + self.pipeline.validate() + return self.pipeline + + def load_yaml( + self, base_dir: Path, path: Path, environment: str | None ) -> Pipeline: """Load pipeline definition from yaml. @@ -130,15 +136,12 @@ def load_from_yaml( :param base_dir: Base directory to the pipelines (default is current working directory) :param path: Path to pipeline definition yaml file - :param registry: Pipeline components registry - :param config: Pipeline config - :param handlers: Component handlers :raises TypeError: The pipeline definition should contain a list of components :raises TypeError: The env-specific pipeline definition should contain a list of components :returns: Initialized pipeline object """ - Pipeline.set_pipeline_name_env_vars(base_dir, path) - Pipeline.set_environment_name(environment) + PipelineGenerator.set_pipeline_name_env_vars(base_dir, path) + PipelineGenerator.set_environment_name(environment) main_content = load_yaml_file(path, substitution=ENV) if not isinstance(main_content, list): @@ -148,7 +151,9 @@ def load_from_yaml( if ( environment and ( - env_file := Pipeline.pipeline_filename_environment(path, environment) + env_file := PipelineGenerator.pipeline_filename_environment( + path, environment + ) ).exists() ): env_content = load_yaml_file(env_file, substitution=ENV) @@ -156,17 +161,17 @@ def load_from_yaml( msg = f"The pipeline definition {env_file} should contain a list of components" raise TypeError(msg) - return cls(main_content, env_content, registry, config, handlers) + return self.parse(main_content, env_content) - def parse_components(self, component_list: list[dict]) -> None: + def parse_components(self, components: list[dict]) -> None: """Instantiate, enrich and inflate a list of components. - :param component_list: List of components + :param components: List of components :raises ValueError: Every component must have a type defined :raises ParsingException: Error enriching component :raises ParsingException: All undefined exceptions """ - for component_data in component_list: + for component_data in components: try: try: component_type: str = component_data["type"] @@ -208,21 +213,21 @@ def apply_component( original_from_component_name, from_topic, ) in enriched_component.from_.components.items(): - original_from_component = self.components.find( + original_from_component = self.pipeline.find( original_from_component_name ) inflated_from_component = original_from_component.inflate()[-1] - resolved_from_component = self.components.find( + resolved_from_component = self.pipeline.find( inflated_from_component.name ) enriched_component.weave_from_topics( resolved_from_component.to, from_topic ) - elif self.components: + elif self.pipeline: # read from previous component - prev_component = self.components.last + prev_component = self.pipeline.last enriched_component.weave_from_topics(prev_component.to) - self.components.add(enriched_component) + self.pipeline.add(enriched_component) def enrich_component( self, @@ -251,32 +256,6 @@ def enrich_component( **component_data, ) - def print_yaml(self, substitution: dict | None = None) -> None: - """Print the generated pipeline definition. - - :param substitution: Substitution dictionary, defaults to None - """ - syntax = Syntax( - substitute(str(self), substitution), - "yaml", - background_color="default", - theme="ansi_dark", - ) - Console( - width=1000 # HACK: overwrite console width to avoid truncating output - ).print(syntax) - - def __iter__(self) -> Iterator[PipelineComponent]: - return iter(self.components) - - def __str__(self) -> str: - return yaml.dump( - self.components.model_dump(mode="json", by_alias=True, exclude_none=True) - ) - - def __len__(self) -> int: - return len(self.components) - def substitute_in_component(self, component_as_dict: dict) -> dict: """Substitute all $-placeholders in a component in dict representation. @@ -310,15 +289,12 @@ def substitute_in_component(self, component_as_dict: dict) -> dict: ) ) - def validate(self) -> None: - self.components.validate_unique_names() - @staticmethod def pipeline_filename_environment(pipeline_path: Path, environment: str) -> Path: """Add the environment name from the KpopsConfig to the pipeline.yaml path. :param pipeline_path: Path to pipeline.yaml file - :param config: The KpopsConfig + :param environment: Environment name :returns: An absolute path to the pipeline_.yaml """ return pipeline_path.with_stem(f"{pipeline_path.stem}_{environment}") diff --git a/kpops/pipeline_generator/__init__.py b/kpops/pipeline_generator/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kpops/utils/pydantic.py b/kpops/utils/pydantic.py index 8d8a55493..3b643af51 100644 --- a/kpops/utils/pydantic.py +++ b/kpops/utils/pydantic.py @@ -11,7 +11,7 @@ from kpops.utils.dict_ops import update_nested_pair from kpops.utils.docstring import describe_object -from kpops.utils.yaml_loading import load_yaml_file +from kpops.utils.yaml import load_yaml_file def to_camel(s: str) -> str: diff --git a/kpops/utils/yaml_loading.py b/kpops/utils/yaml.py similarity index 83% rename from kpops/utils/yaml_loading.py rename to kpops/utils/yaml.py index 668a609cc..2f0e16608 100644 --- a/kpops/utils/yaml_loading.py +++ b/kpops/utils/yaml.py @@ -5,6 +5,8 @@ import yaml from cachetools import cached from cachetools.keys import hashkey +from rich.console import Console +from rich.syntax import Syntax from kpops.utils.dict_ops import ImprovedTemplate @@ -79,3 +81,20 @@ def substitute_nested(input: str, **kwargs) -> str: msg = "An infinite loop condition detected. Check substitution variables." raise ValueError(msg) return old_str + + +def print_yaml(input: str, *, substitution: dict | None = None) -> None: + """Print YAML to console with syntax highlighting. + + :param s: YAML content + :param substitution: Substitution dictionary, defaults to None + """ + syntax = Syntax( + substitute(input, substitution), + "yaml", + background_color="default", + theme="ansi_dark", + ) + Console( + width=1000 # HACK: overwrite console width to avoid truncating output + ).print(syntax) diff --git a/tests/cli/test_pipeline_steps.py b/tests/cli/test_pipeline_steps.py index a09d7b064..f9a345ae7 100644 --- a/tests/cli/test_pipeline_steps.py +++ b/tests/cli/test_pipeline_steps.py @@ -6,7 +6,8 @@ from pytest_mock import MockerFixture from kpops.cli.main import FilterType, get_steps_to_apply -from kpops.pipeline_generator.pipeline import Pipeline +from kpops.components import PipelineComponent +from kpops.pipeline import Pipeline PREFIX = "example-prefix-" @@ -25,17 +26,11 @@ class TestComponent: @pytest.fixture(autouse=True) def pipeline() -> Pipeline: - class TestPipeline: - components = [ - test_component_1, - test_component_2, - test_component_3, - ] - - def __iter__(self): - return iter(self.components) - - return cast(Pipeline, TestPipeline()) + pipeline = Pipeline() + pipeline.add(cast(PipelineComponent, test_component_1)) + pipeline.add(cast(PipelineComponent, test_component_2)) + pipeline.add(cast(PipelineComponent, test_component_3)) + return pipeline @pytest.fixture(autouse=True) diff --git a/tests/compiler/test_pipeline_name.py b/tests/compiler/test_pipeline_name.py index 5ca0da6ee..cca9fe88c 100644 --- a/tests/compiler/test_pipeline_name.py +++ b/tests/compiler/test_pipeline_name.py @@ -2,7 +2,7 @@ import pytest -from kpops.pipeline_generator.pipeline import Pipeline +from kpops.pipeline import PipelineGenerator from kpops.utils.environment import ENV DEFAULTS_PATH = Path(__file__).parent / "resources" @@ -11,7 +11,7 @@ def test_should_set_pipeline_name_with_default_base_dir(): - Pipeline.set_pipeline_name_env_vars(DEFAULT_BASE_DIR, PIPELINE_PATH) + PipelineGenerator.set_pipeline_name_env_vars(DEFAULT_BASE_DIR, PIPELINE_PATH) assert ENV["pipeline_name"] == "some-random-path-for-testing" assert ENV["pipeline_name_0"] == "some" @@ -22,7 +22,9 @@ def test_should_set_pipeline_name_with_default_base_dir(): def test_should_set_pipeline_name_with_specific_relative_base_dir(): - Pipeline.set_pipeline_name_env_vars(Path("./some/random/path"), PIPELINE_PATH) + PipelineGenerator.set_pipeline_name_env_vars( + Path("./some/random/path"), PIPELINE_PATH + ) assert ENV["pipeline_name"] == "for-testing" assert ENV["pipeline_name_0"] == "for" @@ -30,7 +32,9 @@ def test_should_set_pipeline_name_with_specific_relative_base_dir(): def test_should_set_pipeline_name_with_specific_absolute_base_dir(): - Pipeline.set_pipeline_name_env_vars(Path("some/random/path"), PIPELINE_PATH) + PipelineGenerator.set_pipeline_name_env_vars( + Path("some/random/path"), PIPELINE_PATH + ) assert ENV["pipeline_name"] == "for-testing" assert ENV["pipeline_name_0"] == "for" @@ -38,7 +42,7 @@ def test_should_set_pipeline_name_with_specific_absolute_base_dir(): def test_should_set_pipeline_name_with_absolute_base_dir(): - Pipeline.set_pipeline_name_env_vars(Path.cwd(), PIPELINE_PATH) + PipelineGenerator.set_pipeline_name_env_vars(Path.cwd(), PIPELINE_PATH) assert ENV["pipeline_name"] == "some-random-path-for-testing" assert ENV["pipeline_name_0"] == "some" @@ -52,11 +56,11 @@ def test_should_not_set_pipeline_name_with_the_same_base_dir(): with pytest.raises( ValueError, match="The pipeline-base-dir should not equal the pipeline-path" ): - Pipeline.set_pipeline_name_env_vars(PIPELINE_PATH, PIPELINE_PATH) + PipelineGenerator.set_pipeline_name_env_vars(PIPELINE_PATH, PIPELINE_PATH) def test_pipeline_file_name_environment(): - environment = Pipeline.pipeline_filename_environment( + environment = PipelineGenerator.pipeline_filename_environment( PIPELINE_PATH, "some_environment" ) assert environment.name == "pipeline_some_environment.yaml" diff --git a/tests/compiler/test_yaml_loading.py b/tests/compiler/test_yaml_loading.py index c4bdcc3cf..47db51bfd 100644 --- a/tests/compiler/test_yaml_loading.py +++ b/tests/compiler/test_yaml_loading.py @@ -4,7 +4,7 @@ import pytest import yaml -from kpops.utils.yaml_loading import load_yaml_file +from kpops.utils.yaml import load_yaml_file RESOURCE_PATH = Path(__file__).parent / "resources" diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index fe478e7b0..e449da9a5 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -137,11 +137,11 @@ def test_inherit(self, config: KpopsConfig, handlers: ComponentHandlers): component = Child( config=config, handlers=handlers, - name="name-defined-in-pipeline_generator", + name="name-defined-in-pipeline_parser", ) assert ( - component.name == "name-defined-in-pipeline_generator" + component.name == "name-defined-in-pipeline_parser" ), "Kwargs should should overwrite all other values" assert component.nice == { "fake-value": "fake" diff --git a/tests/conftest.py b/tests/conftest.py index 9da841b07..479672e86 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import pytest -from kpops.utils.yaml_loading import load_yaml_file +from kpops.utils.yaml import load_yaml_file @pytest.fixture() diff --git a/tests/pipeline/snapshots/snap_test_example.py b/tests/pipeline/snapshots/snap_test_example.py index 406679c8b..14d3d650c 100644 --- a/tests/pipeline/snapshots/snap_test_example.py +++ b/tests/pipeline/snapshots/snap_test_example.py @@ -7,348 +7,346 @@ snapshots = Snapshot() -snapshots['TestExample.test_atm_fraud atm-fraud-pipeline'] = { - 'components': [ - { - 'app': { - 'debug': True, - 'image': '${DOCKER_REGISTRY}/atm-demo-accountproducer', - 'imageTag': '1.0.0', - 'nameOverride': 'account-producer', - 'prometheus': { - 'jmx': { - 'enabled': False - } - }, - 'replicaCount': 1, - 'schedule': '0 12 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'optimizeLeaveGroupBehavior': False, - 'outputTopic': 'bakdata-atm-fraud-detection-account-producer-topic', - 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' - }, - 'suspend': True +snapshots['TestExample.test_atm_fraud atm-fraud-pipeline'] = [ + { + 'app': { + 'debug': True, + 'image': '${DOCKER_REGISTRY}/atm-demo-accountproducer', + 'imageTag': '1.0.0', + 'nameOverride': 'account-producer', + 'prometheus': { + 'jmx': { + 'enabled': False + } }, - 'name': 'account-producer', - 'namespace': '${NAMESPACE}', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False + 'replicaCount': 1, + 'schedule': '0 12 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' + 'optimizeLeaveGroupBehavior': False, + 'outputTopic': 'bakdata-atm-fraud-detection-account-producer-topic', + 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' }, - 'to': { - 'models': { - }, - 'topics': { - 'bakdata-atm-fraud-detection-account-producer-topic': { - 'configs': { - }, - 'partitions_count': 3 - } - } + 'suspend': True + }, + 'name': 'account-producer', + 'namespace': '${NAMESPACE}', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False }, - 'type': 'producer-app', - 'version': '2.9.0' + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' }, - { - 'app': { - 'commandLine': { - 'ITERATION': 20, - 'REAL_TX': 19 - }, - 'debug': True, - 'image': '${DOCKER_REGISTRY}/atm-demo-transactionavroproducer', - 'imageTag': '1.0.0', - 'nameOverride': 'transaction-avro-producer', - 'prometheus': { - 'jmx': { - 'enabled': False - } - }, - 'replicaCount': 1, - 'schedule': '0 12 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { + 'to': { + 'models': { + }, + 'topics': { + 'bakdata-atm-fraud-detection-account-producer-topic': { + 'configs': { }, - 'optimizeLeaveGroupBehavior': False, - 'outputTopic': 'bakdata-atm-fraud-detection-transaction-avro-producer-topic', - 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' - }, - 'suspend': True + 'partitions_count': 3 + } + } + }, + 'type': 'producer-app', + 'version': '2.9.0' + }, + { + 'app': { + 'commandLine': { + 'ITERATION': 20, + 'REAL_TX': 19 }, - 'name': 'transaction-avro-producer', - 'namespace': '${NAMESPACE}', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' + 'debug': True, + 'image': '${DOCKER_REGISTRY}/atm-demo-transactionavroproducer', + 'imageTag': '1.0.0', + 'nameOverride': 'transaction-avro-producer', + 'prometheus': { + 'jmx': { + 'enabled': False + } }, - 'to': { - 'models': { + 'replicaCount': 1, + 'schedule': '0 12 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { }, - 'topics': { - 'bakdata-atm-fraud-detection-transaction-avro-producer-topic': { - 'configs': { - }, - 'partitions_count': 3 - } - } + 'optimizeLeaveGroupBehavior': False, + 'outputTopic': 'bakdata-atm-fraud-detection-transaction-avro-producer-topic', + 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' }, - 'type': 'producer-app', - 'version': '2.9.0' + 'suspend': True }, - { - 'app': { - 'annotations': { - 'consumerGroup': 'atm-transactionjoiner-atm-fraud-joinedtransactions-topic' - }, - 'commandLine': { - 'PRODUCTIVE': False - }, - 'debug': True, - 'image': '${DOCKER_REGISTRY}/atm-demo-transactionjoiner', - 'imageTag': '1.0.0', - 'labels': { - 'pipeline': 'bakdata-atm-fraud-detection' - }, - 'nameOverride': 'transaction-joiner', - 'prometheus': { - 'jmx': { - 'enabled': False - } - }, - 'replicaCount': 1, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'errorTopic': 'bakdata-atm-fraud-detection-transaction-joiner-dead-letter-topic', - 'inputTopics': [ - 'bakdata-atm-fraud-detection-transaction-avro-producer-topic' - ], - 'optimizeLeaveGroupBehavior': False, - 'outputTopic': 'bakdata-atm-fraud-detection-transaction-joiner-topic', - 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' - } + 'name': 'transaction-avro-producer', + 'namespace': '${NAMESPACE}', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False }, - 'name': 'transaction-joiner', - 'namespace': '${NAMESPACE}', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'to': { - 'models': { - }, - 'topics': { - 'bakdata-atm-fraud-detection-transaction-joiner-dead-letter-topic': { - 'configs': { - }, - 'partitions_count': 1, - 'type': 'error' + 'topics': { + 'bakdata-atm-fraud-detection-transaction-avro-producer-topic': { + 'configs': { }, - 'bakdata-atm-fraud-detection-transaction-joiner-topic': { - 'configs': { - }, - 'partitions_count': 3 - } + 'partitions_count': 3 } - }, - 'type': 'streams-app', - 'version': '2.9.0' + } }, - { - 'app': { - 'annotations': { - 'consumerGroup': 'atm-frauddetector-atm-fraud-possiblefraudtransactions-topic' - }, - 'commandLine': { - 'PRODUCTIVE': False - }, - 'debug': True, - 'image': '${DOCKER_REGISTRY}/atm-demo-frauddetector', - 'imageTag': '1.0.0', - 'labels': { - 'pipeline': 'bakdata-atm-fraud-detection' - }, - 'nameOverride': 'fraud-detector', - 'prometheus': { - 'jmx': { - 'enabled': False - } - }, - 'replicaCount': 1, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'errorTopic': 'bakdata-atm-fraud-detection-fraud-detector-dead-letter-topic', - 'inputTopics': [ - 'bakdata-atm-fraud-detection-transaction-joiner-topic' - ], - 'optimizeLeaveGroupBehavior': False, - 'outputTopic': 'bakdata-atm-fraud-detection-fraud-detector-topic', - 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' + 'type': 'producer-app', + 'version': '2.9.0' + }, + { + 'app': { + 'annotations': { + 'consumerGroup': 'atm-transactionjoiner-atm-fraud-joinedtransactions-topic' + }, + 'commandLine': { + 'PRODUCTIVE': False + }, + 'debug': True, + 'image': '${DOCKER_REGISTRY}/atm-demo-transactionjoiner', + 'imageTag': '1.0.0', + 'labels': { + 'pipeline': 'bakdata-atm-fraud-detection' + }, + 'nameOverride': 'transaction-joiner', + 'prometheus': { + 'jmx': { + 'enabled': False } }, - 'name': 'fraud-detector', - 'namespace': '${NAMESPACE}', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' + 'replicaCount': 1, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'errorTopic': 'bakdata-atm-fraud-detection-transaction-joiner-dead-letter-topic', + 'inputTopics': [ + 'bakdata-atm-fraud-detection-transaction-avro-producer-topic' + ], + 'optimizeLeaveGroupBehavior': False, + 'outputTopic': 'bakdata-atm-fraud-detection-transaction-joiner-topic', + 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' + } + }, + 'name': 'transaction-joiner', + 'namespace': '${NAMESPACE}', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False }, - 'to': { - 'models': { + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { + }, + 'topics': { + 'bakdata-atm-fraud-detection-transaction-joiner-dead-letter-topic': { + 'configs': { + }, + 'partitions_count': 1, + 'type': 'error' }, - 'topics': { - 'bakdata-atm-fraud-detection-fraud-detector-dead-letter-topic': { - 'configs': { - }, - 'partitions_count': 1, - 'type': 'error' + 'bakdata-atm-fraud-detection-transaction-joiner-topic': { + 'configs': { }, - 'bakdata-atm-fraud-detection-fraud-detector-topic': { - 'configs': { - }, - 'partitions_count': 3 - } + 'partitions_count': 3 } + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + }, + { + 'app': { + 'annotations': { + 'consumerGroup': 'atm-frauddetector-atm-fraud-possiblefraudtransactions-topic' + }, + 'commandLine': { + 'PRODUCTIVE': False + }, + 'debug': True, + 'image': '${DOCKER_REGISTRY}/atm-demo-frauddetector', + 'imageTag': '1.0.0', + 'labels': { + 'pipeline': 'bakdata-atm-fraud-detection' }, - 'type': 'streams-app', - 'version': '2.9.0' + 'nameOverride': 'fraud-detector', + 'prometheus': { + 'jmx': { + 'enabled': False + } + }, + 'replicaCount': 1, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'errorTopic': 'bakdata-atm-fraud-detection-fraud-detector-dead-letter-topic', + 'inputTopics': [ + 'bakdata-atm-fraud-detection-transaction-joiner-topic' + ], + 'optimizeLeaveGroupBehavior': False, + 'outputTopic': 'bakdata-atm-fraud-detection-fraud-detector-topic', + 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' + } }, - { - 'app': { - 'annotations': { - 'consumerGroup': 'atm-accountlinker-atm-fraud-output-topic' - }, - 'commandLine': { - 'PRODUCTIVE': False - }, - 'debug': True, - 'image': '${DOCKER_REGISTRY}/atm-demo-accountlinker', - 'imageTag': '1.0.0', - 'labels': { - 'pipeline': 'bakdata-atm-fraud-detection' - }, - 'nameOverride': 'account-linker', - 'prometheus': { - 'jmx': { - 'enabled': False - } + 'name': 'fraud-detector', + 'namespace': '${NAMESPACE}', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { + }, + 'topics': { + 'bakdata-atm-fraud-detection-fraud-detector-dead-letter-topic': { + 'configs': { + }, + 'partitions_count': 1, + 'type': 'error' }, - 'replicaCount': 1, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'errorTopic': 'bakdata-atm-fraud-detection-account-linker-dead-letter-topic', - 'extraInputTopics': { - 'accounts': [ - 'bakdata-atm-fraud-detection-account-producer-topic' - ] + 'bakdata-atm-fraud-detection-fraud-detector-topic': { + 'configs': { }, - 'inputTopics': [ - 'bakdata-atm-fraud-detection-fraud-detector-topic' - ], - 'optimizeLeaveGroupBehavior': False, - 'outputTopic': 'bakdata-atm-fraud-detection-account-linker-topic', - 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' + 'partitions_count': 3 } + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + }, + { + 'app': { + 'annotations': { + 'consumerGroup': 'atm-accountlinker-atm-fraud-output-topic' }, - 'from': { - 'components': { - 'account-producer': { - 'role': 'accounts' - }, - 'fraud-detector': { - 'type': 'input' - } - }, - 'topics': { + 'commandLine': { + 'PRODUCTIVE': False + }, + 'debug': True, + 'image': '${DOCKER_REGISTRY}/atm-demo-accountlinker', + 'imageTag': '1.0.0', + 'labels': { + 'pipeline': 'bakdata-atm-fraud-detection' + }, + 'nameOverride': 'account-linker', + 'prometheus': { + 'jmx': { + 'enabled': False } }, - 'name': 'account-linker', - 'namespace': '${NAMESPACE}', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False + 'replicaCount': 1, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'errorTopic': 'bakdata-atm-fraud-detection-account-linker-dead-letter-topic', + 'extraInputTopics': { + 'accounts': [ + 'bakdata-atm-fraud-detection-account-producer-topic' + ] }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { + 'inputTopics': [ + 'bakdata-atm-fraud-detection-fraud-detector-topic' + ], + 'optimizeLeaveGroupBehavior': False, + 'outputTopic': 'bakdata-atm-fraud-detection-account-linker-topic', + 'schemaRegistryUrl': 'http://k8kafka-cp-schema-registry.kpops.svc.cluster.local:8081/' + } + }, + 'from': { + 'components': { + 'account-producer': { + 'role': 'accounts' }, - 'topics': { - 'bakdata-atm-fraud-detection-account-linker-dead-letter-topic': { - 'configs': { - }, - 'partitions_count': 1, - 'type': 'error' - }, - 'bakdata-atm-fraud-detection-account-linker-topic': { - 'configs': { - }, - 'partitions_count': 3 - } + 'fraud-detector': { + 'type': 'input' } }, - 'type': 'streams-app', - 'version': '2.9.0' + 'topics': { + } }, - { - 'app': { - 'auto.create': True, - 'connection.ds.pool.size': 5, - 'connection.password': 'AppPassword', - 'connection.url': 'jdbc:postgresql://postgresql-dev.${NAMESPACE}.svc.cluster.local:5432/app_db', - 'connection.user': 'app1', - 'connector.class': 'io.confluent.connect.jdbc.JdbcSinkConnector', - 'errors.deadletterqueue.context.headers.enable': True, - 'errors.deadletterqueue.topic.name': 'postgres-request-sink-dead-letters', - 'errors.deadletterqueue.topic.replication.factor': 1, - 'errors.tolerance': 'all', - 'insert.mode': 'insert', - 'insert.mode.databaselevel': True, - 'key.converter': 'org.apache.kafka.connect.storage.StringConverter', - 'name': 'postgresql-connector', - 'pk.mode': 'record_value', - 'table.name.format': 'fraud_transactions', - 'tasks.max': 1, - 'topics': 'bakdata-atm-fraud-detection-account-linker-topic', - 'transforms': 'flatten', - 'transforms.flatten.type': 'org.apache.kafka.connect.transforms.Flatten$Value', - 'value.converter': 'io.confluent.connect.avro.AvroConverter', - 'value.converter.schema.registry.url': 'http://k8kafka-cp-schema-registry.${NAMESPACE}.svc.cluster.local:8081' + 'name': 'account-linker', + 'namespace': '${NAMESPACE}', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False }, - 'name': 'postgresql-connector', - 'namespace': '${NAMESPACE}', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-kafka-connect-resetter', - 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'resetter_values': { + 'topics': { + 'bakdata-atm-fraud-detection-account-linker-dead-letter-topic': { + 'configs': { + }, + 'partitions_count': 1, + 'type': 'error' + }, + 'bakdata-atm-fraud-detection-account-linker-topic': { + 'configs': { + }, + 'partitions_count': 3 + } + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + }, + { + 'app': { + 'auto.create': True, + 'connection.ds.pool.size': 5, + 'connection.password': 'AppPassword', + 'connection.url': 'jdbc:postgresql://postgresql-dev.${NAMESPACE}.svc.cluster.local:5432/app_db', + 'connection.user': 'app1', + 'connector.class': 'io.confluent.connect.jdbc.JdbcSinkConnector', + 'errors.deadletterqueue.context.headers.enable': True, + 'errors.deadletterqueue.topic.name': 'postgres-request-sink-dead-letters', + 'errors.deadletterqueue.topic.replication.factor': 1, + 'errors.tolerance': 'all', + 'insert.mode': 'insert', + 'insert.mode.databaselevel': True, + 'key.converter': 'org.apache.kafka.connect.storage.StringConverter', + 'name': 'postgresql-connector', + 'pk.mode': 'record_value', + 'table.name.format': 'fraud_transactions', + 'tasks.max': 1, + 'topics': 'bakdata-atm-fraud-detection-account-linker-topic', + 'transforms': 'flatten', + 'transforms.flatten.type': 'org.apache.kafka.connect.transforms.Flatten$Value', + 'value.converter': 'io.confluent.connect.avro.AvroConverter', + 'value.converter.schema.registry.url': 'http://k8kafka-cp-schema-registry.${NAMESPACE}.svc.cluster.local:8081' + }, + 'name': 'postgresql-connector', + 'namespace': '${NAMESPACE}', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False }, - 'type': 'kafka-sink-connector', - 'version': '1.0.4' - } - ] -} + 'repository_name': 'bakdata-kafka-connect-resetter', + 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + }, + 'resetter_values': { + }, + 'type': 'kafka-sink-connector', + 'version': '1.0.4' + } +] diff --git a/tests/pipeline/snapshots/snap_test_pipeline.py b/tests/pipeline/snapshots/snap_test_pipeline.py index d1e6f1776..c9fee4d4b 100644 --- a/tests/pipeline/snapshots/snap_test_pipeline.py +++ b/tests/pipeline/snapshots/snap_test_pipeline.py @@ -7,2303 +7,2275 @@ snapshots = Snapshot() -snapshots['TestPipeline.test_default_config test-pipeline'] = { - 'components': [ - { - 'app': { - 'nameOverride': 'resources-custom-config-app1', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-custom-config-app1', - 'schemaRegistryUrl': 'http://localhost:8081/' +snapshots['TestPipeline.test_default_config test-pipeline'] = [ + { + 'app': { + 'nameOverride': 'resources-custom-config-app1', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' } }, - 'name': 'app1', - 'namespace': 'development-namespace', - 'prefix': 'resources-custom-config-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-custom-config-app1': { - 'configs': { - }, - 'partitions_count': 3, - 'type': 'output' - } - } + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-custom-config-app1', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app1', + 'namespace': 'development-namespace', + 'prefix': 'resources-custom-config-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'producer-app', - 'version': '2.9.0' - }, - { - 'app': { - 'image': 'some-image', - 'labels': { - 'pipeline': 'resources-custom-config' - }, - 'nameOverride': 'resources-custom-config-app2', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'errorTopic': 'resources-custom-config-app2-error', - 'inputTopics': [ - 'resources-custom-config-app1' - ], - 'outputTopic': 'resources-custom-config-app2', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-custom-config-app1': { + 'configs': { + }, + 'partitions_count': 3, + 'type': 'output' } + } + }, + 'type': 'producer-app', + 'version': '2.9.0' + }, + { + 'app': { + 'image': 'some-image', + 'labels': { + 'pipeline': 'resources-custom-config' + }, + 'nameOverride': 'resources-custom-config-app2', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'errorTopic': 'resources-custom-config-app2-error', + 'inputTopics': [ + 'resources-custom-config-app1' + ], + 'outputTopic': 'resources-custom-config-app2', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app2', + 'namespace': 'development-namespace', + 'prefix': 'resources-custom-config-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'app2', - 'namespace': 'development-namespace', - 'prefix': 'resources-custom-config-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-custom-config-app2': { - 'configs': { - }, - 'partitions_count': 3, - 'type': 'output' - }, - 'resources-custom-config-app2-error': { - 'configs': { - }, - 'partitions_count': 1, - 'type': 'error' - } + 'topics': { + 'resources-custom-config-app2': { + 'configs': { + }, + 'partitions_count': 3, + 'type': 'output' + }, + 'resources-custom-config-app2-error': { + 'configs': { + }, + 'partitions_count': 1, + 'type': 'error' } - }, - 'type': 'streams-app', - 'version': '2.9.0' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + } +] -snapshots['TestPipeline.test_inflate_pipeline test-pipeline'] = { - 'components': [ - { - 'app': { - 'commandLine': { - 'FAKE_ARG': 'fake-arg-value' - }, - 'image': 'example-registry/fake-image', - 'imageTag': '0.0.1', - 'nameOverride': 'resources-pipeline-with-inflate-scheduled-producer', - 'schedule': '30 3/8 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-pipeline-with-inflate-scheduled-producer', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'scheduled-producer', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-inflate-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - 'com/bakdata/kafka/fake': '1.0.0' - }, - 'topics': { - 'resources-pipeline-with-inflate-scheduled-producer': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 12, - 'type': 'output', - 'value_schema': 'com.bakdata.fake.Produced' - } - } - }, - 'type': 'scheduled-producer', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'converter-resources-pipeline-with-inflate-converter', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 1, - 'minReplicas': 0, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - ] - }, - 'commandLine': { - 'CONVERT_XML': True - }, - 'nameOverride': 'resources-pipeline-with-inflate-converter', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-pipeline-with-inflate-converter-error', - 'inputTopics': [ - 'resources-pipeline-with-inflate-scheduled-producer' - ], - 'outputTopic': 'resources-pipeline-with-inflate-converter', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'converter', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-inflate-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-pipeline-with-inflate-converter': { - 'configs': { - 'cleanup.policy': 'compact,delete', - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-pipeline-with-inflate-converter-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 10, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } - }, - 'type': 'converter', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'filter-resources-pipeline-with-inflate-should-inflate', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 4, - 'minReplicas': 4, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - 'resources-pipeline-with-inflate-should-inflate' - ] - }, - 'commandLine': { - 'TYPE': 'nothing' - }, - 'image': 'fake-registry/filter', - 'imageTag': '2.4.1', - 'nameOverride': 'resources-pipeline-with-inflate-should-inflate', - 'replicaCount': 4, - 'resources': { - 'requests': { - 'memory': '3G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-pipeline-with-inflate-should-inflate-error', - 'inputTopics': [ - 'resources-pipeline-with-inflate-converter' - ], - 'outputTopic': 'resources-pipeline-with-inflate-should-inflate', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'should-inflate', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-inflate-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-pipeline-with-inflate-should-inflate': { - 'configs': { - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-pipeline-with-inflate-should-inflate-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } - }, - 'type': 'should-inflate', - 'version': '2.4.2' - }, - { - 'app': { - 'batch.size': '2000', - 'behavior.on.malformed.documents': 'warn', - 'behavior.on.null.values': 'delete', - 'connection.compression': 'true', - 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', - 'key.ignore': 'false', - 'linger.ms': '5000', - 'max.buffered.records': '20000', - 'name': 'resources-pipeline-with-inflate-should-inflate-inflated-sink-connector', - 'read.timeout.ms': '120000', - 'tasks.max': '1', - 'topics': 'resources-pipeline-with-inflate-should-inflate', - 'transforms.changeTopic.replacement': 'resources-pipeline-with-inflate-should-inflate-index-v1' - }, - 'name': 'should-inflate-inflated-sink-connector', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-inflate-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-kafka-connect-resetter', - 'url': 'https://bakdata.github.io/kafka-connect-resetter/' - }, - 'resetter_values': { - }, - 'to': { - 'models': { - }, - 'topics': { - 'kafka-sink-connector': { - 'configs': { - }, - 'type': 'output' - }, - 'should-inflate-inflated-sink-connector': { - 'configs': { - }, - 'role': 'test' - } - } +snapshots['TestPipeline.test_inflate_pipeline test-pipeline'] = [ + { + 'app': { + 'commandLine': { + 'FAKE_ARG': 'fake-arg-value' + }, + 'image': 'example-registry/fake-image', + 'imageTag': '0.0.1', + 'nameOverride': 'resources-pipeline-with-inflate-scheduled-producer', + 'schedule': '30 3/8 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-pipeline-with-inflate-scheduled-producer', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'scheduled-producer', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-inflate-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { + 'com/bakdata/kafka/fake': '1.0.0' }, - 'type': 'kafka-sink-connector', - 'version': '1.0.4' - }, - { - 'app': { - 'nameOverride': 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app-error', - 'inputTopics': [ - 'kafka-sink-connector' - ], - 'outputTopic': 'resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-pipeline-with-inflate-scheduled-producer': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 12, + 'type': 'output', + 'value_schema': 'com.bakdata.fake.Produced' } + } + }, + 'type': 'scheduled-producer', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'converter-resources-pipeline-with-inflate-converter', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 1, + 'minReplicas': 0, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + ] + }, + 'commandLine': { + 'CONVERT_XML': True + }, + 'nameOverride': 'resources-pipeline-with-inflate-converter', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-pipeline-with-inflate-converter-error', + 'inputTopics': [ + 'resources-pipeline-with-inflate-scheduled-producer' + ], + 'outputTopic': 'resources-pipeline-with-inflate-converter', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'converter', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-inflate-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'should-inflate-inflated-streams-app', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-inflate-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - }, - 'resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app': { - 'configs': { - }, - 'type': 'output' - } + 'topics': { + 'resources-pipeline-with-inflate-converter': { + 'configs': { + 'cleanup.policy': 'compact,delete', + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-pipeline-with-inflate-converter-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 10, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'converter', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'filter-resources-pipeline-with-inflate-should-inflate', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 4, + 'minReplicas': 4, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + 'resources-pipeline-with-inflate-should-inflate' + ] + }, + 'commandLine': { + 'TYPE': 'nothing' + }, + 'image': 'fake-registry/filter', + 'imageTag': '2.4.1', + 'nameOverride': 'resources-pipeline-with-inflate-should-inflate', + 'replicaCount': 4, + 'resources': { + 'requests': { + 'memory': '3G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-pipeline-with-inflate-should-inflate-error', + 'inputTopics': [ + 'resources-pipeline-with-inflate-converter' + ], + 'outputTopic': 'resources-pipeline-with-inflate-should-inflate', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'should-inflate', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-inflate-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'streams-app', - 'version': '2.4.2' - } - ] -} - -snapshots['TestPipeline.test_kafka_connect_sink_weave_from_topics test-pipeline'] = { - 'components': [ - { - 'app': { - 'image': 'fake-image', - 'nameOverride': 'resources-kafka-connect-sink-streams-app', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-kafka-connect-sink-streams-app-error', - 'inputTopics': [ - 'example-topic' - ], - 'outputTopic': 'example-output', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-pipeline-with-inflate-should-inflate': { + 'configs': { + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-pipeline-with-inflate-should-inflate-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'should-inflate', + 'version': '2.4.2' + }, + { + 'app': { + 'batch.size': '2000', + 'behavior.on.malformed.documents': 'warn', + 'behavior.on.null.values': 'delete', + 'connection.compression': 'true', + 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', + 'key.ignore': 'false', + 'linger.ms': '5000', + 'max.buffered.records': '20000', + 'name': 'resources-pipeline-with-inflate-should-inflate-inflated-sink-connector', + 'read.timeout.ms': '120000', + 'tasks.max': '1', + 'topics': 'resources-pipeline-with-inflate-should-inflate', + 'transforms.changeTopic.replacement': 'resources-pipeline-with-inflate-should-inflate-index-v1' + }, + 'name': 'should-inflate-inflated-sink-connector', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-inflate-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-kafka-connect-resetter', + 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + }, + 'resetter_values': { + }, + 'to': { + 'models': { }, - 'from': { - 'components': { + 'topics': { + 'kafka-sink-connector': { + 'configs': { + }, + 'type': 'output' }, - 'topics': { - 'example-topic': { - 'type': 'input' - } + 'should-inflate-inflated-sink-connector': { + 'configs': { + }, + 'role': 'test' } + } + }, + 'type': 'kafka-sink-connector', + 'version': '1.0.4' + }, + { + 'app': { + 'nameOverride': 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app-error', + 'inputTopics': [ + 'kafka-sink-connector' + ], + 'outputTopic': 'resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'should-inflate-inflated-streams-app', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-inflate-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'streams-app', - 'namespace': 'example-namespace', - 'prefix': 'resources-kafka-connect-sink-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'example-output': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-kafka-connect-sink-streams-app-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-pipeline-with-inflate-should-inflate-inflated-streams-app-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' + }, + 'resources-pipeline-with-inflate-should-inflate-should-inflate-inflated-streams-app': { + 'configs': { + }, + 'type': 'output' } - }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'batch.size': '2000', - 'behavior.on.malformed.documents': 'warn', - 'behavior.on.null.values': 'delete', - 'connection.compression': 'true', - 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', - 'key.ignore': 'false', - 'linger.ms': '5000', - 'max.buffered.records': '20000', - 'name': 'resources-kafka-connect-sink-es-sink-connector', - 'read.timeout.ms': '120000', - 'tasks.max': '1', - 'topics': 'example-output' - }, - 'name': 'es-sink-connector', - 'namespace': 'example-namespace', - 'prefix': 'resources-kafka-connect-sink-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-kafka-connect-resetter', - 'url': 'https://bakdata.github.io/kafka-connect-resetter/' - }, - 'resetter_values': { - }, - 'type': 'kafka-sink-connector', - 'version': '1.0.4' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + } +] -snapshots['TestPipeline.test_load_pipeline test-pipeline'] = { - 'components': [ - { - 'app': { - 'commandLine': { - 'FAKE_ARG': 'fake-arg-value' - }, - 'image': 'example-registry/fake-image', - 'imageTag': '0.0.1', - 'nameOverride': 'resources-first-pipeline-scheduled-producer', - 'schedule': '30 3/8 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-first-pipeline-scheduled-producer', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'scheduled-producer', - 'namespace': 'example-namespace', - 'prefix': 'resources-first-pipeline-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - 'com/bakdata/kafka/fake': '1.0.0' - }, - 'topics': { - 'resources-first-pipeline-scheduled-producer': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 12, - 'type': 'output', - 'value_schema': 'com.bakdata.fake.Produced' - } - } - }, - 'type': 'scheduled-producer', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'converter-resources-first-pipeline-converter', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 1, - 'minReplicas': 0, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - ] - }, - 'commandLine': { - 'CONVERT_XML': True - }, - 'nameOverride': 'resources-first-pipeline-converter', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-first-pipeline-converter-error', - 'inputTopics': [ - 'resources-first-pipeline-scheduled-producer' - ], - 'outputTopic': 'resources-first-pipeline-converter', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'converter', - 'namespace': 'example-namespace', - 'prefix': 'resources-first-pipeline-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-first-pipeline-converter': { - 'configs': { - 'cleanup.policy': 'compact,delete', - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-first-pipeline-converter-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 10, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } +snapshots['TestPipeline.test_kafka_connect_sink_weave_from_topics test-pipeline'] = [ + { + 'app': { + 'image': 'fake-image', + 'nameOverride': 'resources-kafka-connect-sink-streams-app', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-kafka-connect-sink-streams-app-error', + 'inputTopics': [ + 'example-topic' + ], + 'outputTopic': 'example-output', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { }, - 'type': 'converter', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'filter-resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 4, - 'minReplicas': 4, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name' - ] - }, - 'commandLine': { - 'TYPE': 'nothing' - }, - 'image': 'fake-registry/filter', - 'imageTag': '2.4.1', - 'nameOverride': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', - 'replicaCount': 4, - 'resources': { - 'requests': { - 'memory': '3G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error', - 'inputTopics': [ - 'resources-first-pipeline-converter' - ], - 'outputTopic': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'example-topic': { + 'type': 'input' } + } + }, + 'name': 'streams-app', + 'namespace': 'example-namespace', + 'prefix': 'resources-kafka-connect-sink-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', - 'namespace': 'example-namespace', - 'prefix': 'resources-first-pipeline-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name': { - 'configs': { - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'example-output': { + 'configs': { + }, + 'type': 'output' + }, + 'resources-kafka-connect-sink-streams-app-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'type': 'filter', - 'version': '2.4.2' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'batch.size': '2000', + 'behavior.on.malformed.documents': 'warn', + 'behavior.on.null.values': 'delete', + 'connection.compression': 'true', + 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', + 'key.ignore': 'false', + 'linger.ms': '5000', + 'max.buffered.records': '20000', + 'name': 'resources-kafka-connect-sink-es-sink-connector', + 'read.timeout.ms': '120000', + 'tasks.max': '1', + 'topics': 'example-output' + }, + 'name': 'es-sink-connector', + 'namespace': 'example-namespace', + 'prefix': 'resources-kafka-connect-sink-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-kafka-connect-resetter', + 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + }, + 'resetter_values': { + }, + 'type': 'kafka-sink-connector', + 'version': '1.0.4' + } +] -snapshots['TestPipeline.test_model_serialization test-pipeline'] = { - 'components': [ - { - 'app': { - 'nameOverride': 'resources-pipeline-with-paths-account-producer', - 'streams': { - 'brokers': 'test', - 'extraOutputTopics': { - }, - 'outputTopic': 'out', - 'schemaRegistryUrl': 'http://localhost:8081/' - } +snapshots['TestPipeline.test_load_pipeline test-pipeline'] = [ + { + 'app': { + 'commandLine': { + 'FAKE_ARG': 'fake-arg-value' + }, + 'image': 'example-registry/fake-image', + 'imageTag': '0.0.1', + 'nameOverride': 'resources-first-pipeline-scheduled-producer', + 'schedule': '30 3/8 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-first-pipeline-scheduled-producer', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'scheduled-producer', + 'namespace': 'example-namespace', + 'prefix': 'resources-first-pipeline-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { + 'com/bakdata/kafka/fake': '1.0.0' }, - 'name': 'account-producer', - 'namespace': 'test', - 'prefix': 'resources-pipeline-with-paths-', - 'repo_config': { - 'repo_auth_flags': { - 'ca_file': 'my-cert.cert', - 'insecure_skip_tls_verify': False, - 'password': '$CI_JOB_TOKEN', - 'username': 'masked' - }, - 'repository_name': 'masked', - 'url': 'masked' - }, - 'type': 'producer-app', - 'version': '2.4.2' - } - ] -} - -snapshots['TestPipeline.test_no_input_topic test-pipeline'] = { - 'components': [ - { - 'app': { - 'commandLine': { - 'CONVERT_XML': True - }, - 'nameOverride': 'resources-no-input-topic-pipeline-app1', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-no-input-topic-pipeline-app1-error', - 'inputPattern': '.*', - 'outputTopic': 'example-output', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-first-pipeline-scheduled-producer': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 12, + 'type': 'output', + 'value_schema': 'com.bakdata.fake.Produced' } + } + }, + 'type': 'scheduled-producer', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'converter-resources-first-pipeline-converter', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 1, + 'minReplicas': 0, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + ] + }, + 'commandLine': { + 'CONVERT_XML': True + }, + 'nameOverride': 'resources-first-pipeline-converter', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-first-pipeline-converter-error', + 'inputTopics': [ + 'resources-first-pipeline-scheduled-producer' + ], + 'outputTopic': 'resources-first-pipeline-converter', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'converter', + 'namespace': 'example-namespace', + 'prefix': 'resources-first-pipeline-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'from': { - 'components': { + 'topics': { + 'resources-first-pipeline-converter': { + 'configs': { + 'cleanup.policy': 'compact,delete', + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' }, - 'topics': { - '.*': { - 'type': 'pattern' - } - } - }, - 'name': 'app1', - 'namespace': 'example-namespace', - 'prefix': 'resources-no-input-topic-pipeline-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'example-output': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-no-input-topic-pipeline-app1-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } - }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'nameOverride': 'resources-no-input-topic-pipeline-app2', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-no-input-topic-pipeline-app2-error', - 'extraOutputTopics': { - 'extra': 'example-output-extra', - 'test-output': 'test-output-extra' - }, - 'inputTopics': [ - 'example-output' - ], - 'schemaRegistryUrl': 'http://localhost:8081/' + 'resources-first-pipeline-converter-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 10, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'converter', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'filter-resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 4, + 'minReplicas': 4, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name' + ] + }, + 'commandLine': { + 'TYPE': 'nothing' + }, + 'image': 'fake-registry/filter', + 'imageTag': '2.4.1', + 'nameOverride': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', + 'replicaCount': 4, + 'resources': { + 'requests': { + 'memory': '3G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error', + 'inputTopics': [ + 'resources-first-pipeline-converter' + ], + 'outputTopic': 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name', + 'namespace': 'example-namespace', + 'prefix': 'resources-first-pipeline-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'app2', - 'namespace': 'example-namespace', - 'prefix': 'resources-no-input-topic-pipeline-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'example-output-extra': { - 'configs': { - }, - 'role': 'extra' - }, - 'resources-no-input-topic-pipeline-app2-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - }, - 'test-output-extra': { - 'configs': { - }, - 'role': 'test-output' - } + 'topics': { + 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name': { + 'configs': { + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-first-pipeline-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-a-long-name-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'type': 'streams-app', - 'version': '2.4.2' - } - ] -} + } + }, + 'type': 'filter', + 'version': '2.4.2' + } +] + +snapshots['TestPipeline.test_model_serialization test-pipeline'] = [ + { + 'app': { + 'nameOverride': 'resources-pipeline-with-paths-account-producer', + 'streams': { + 'brokers': 'test', + 'extraOutputTopics': { + }, + 'outputTopic': 'out', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'account-producer', + 'namespace': 'test', + 'prefix': 'resources-pipeline-with-paths-', + 'repo_config': { + 'repo_auth_flags': { + 'ca_file': 'my-cert.cert', + 'insecure_skip_tls_verify': False, + 'password': '$CI_JOB_TOKEN', + 'username': 'masked' + }, + 'repository_name': 'masked', + 'url': 'masked' + }, + 'type': 'producer-app', + 'version': '2.4.2' + } +] -snapshots['TestPipeline.test_no_user_defined_components test-pipeline'] = { - 'components': [ - { - 'app': { - 'image': 'fake-image', - 'nameOverride': 'resources-no-user-defined-components-streams-app', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-no-user-defined-components-streams-app-error', - 'inputTopics': [ - 'example-topic' - ], - 'outputTopic': 'example-output', - 'schemaRegistryUrl': 'http://localhost:8081/' +snapshots['TestPipeline.test_no_input_topic test-pipeline'] = [ + { + 'app': { + 'commandLine': { + 'CONVERT_XML': True + }, + 'nameOverride': 'resources-no-input-topic-pipeline-app1', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-no-input-topic-pipeline-app1-error', + 'inputPattern': '.*', + 'outputTopic': 'example-output', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { + }, + 'topics': { + '.*': { + 'type': 'pattern' } + } + }, + 'name': 'app1', + 'namespace': 'example-namespace', + 'prefix': 'resources-no-input-topic-pipeline-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'from': { - 'components': { + 'topics': { + 'example-output': { + 'configs': { + }, + 'type': 'output' }, - 'topics': { - 'example-topic': { - 'type': 'input' - } + 'resources-no-input-topic-pipeline-app1-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'resources-no-input-topic-pipeline-app2', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-no-input-topic-pipeline-app2-error', + 'extraOutputTopics': { + 'extra': 'example-output-extra', + 'test-output': 'test-output-extra' + }, + 'inputTopics': [ + 'example-output' + ], + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app2', + 'namespace': 'example-namespace', + 'prefix': 'resources-no-input-topic-pipeline-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'streams-app', - 'namespace': 'example-namespace', - 'prefix': 'resources-no-user-defined-components-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'example-output': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-no-user-defined-components-streams-app-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'example-output-extra': { + 'configs': { + }, + 'role': 'extra' + }, + 'resources-no-input-topic-pipeline-app2-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' + }, + 'test-output-extra': { + 'configs': { + }, + 'role': 'test-output' } - }, - 'type': 'streams-app', - 'version': '2.4.2' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + } +] -snapshots['TestPipeline.test_pipelines_with_env_values test-pipeline'] = { - 'components': [ - { - 'app': { - 'commandLine': { - 'FAKE_ARG': 'override-arg' - }, - 'image': 'example-registry/fake-image', - 'imageTag': '0.0.1', - 'nameOverride': 'resources-pipeline-with-envs-input-producer', - 'schedule': '20 3/8 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-pipeline-with-envs-input-producer', - 'schemaRegistryUrl': 'http://localhost:8081/' - } +snapshots['TestPipeline.test_no_user_defined_components test-pipeline'] = [ + { + 'app': { + 'image': 'fake-image', + 'nameOverride': 'resources-no-user-defined-components-streams-app', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-no-user-defined-components-streams-app-error', + 'inputTopics': [ + 'example-topic' + ], + 'outputTopic': 'example-output', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { }, - 'name': 'input-producer', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-envs-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - 'com/bakdata/kafka/fake': '1.0.0' - }, - 'topics': { - 'resources-pipeline-with-envs-input-producer': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 12, - 'type': 'output', - 'value_schema': 'com.bakdata.fake.Produced' - } + 'topics': { + 'example-topic': { + 'type': 'input' } + } + }, + 'name': 'streams-app', + 'namespace': 'example-namespace', + 'prefix': 'resources-no-user-defined-components-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'scheduled-producer', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'converter-resources-pipeline-with-envs-converter', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 1, - 'minReplicas': 0, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - ] - }, - 'commandLine': { - 'CONVERT_XML': True - }, - 'nameOverride': 'resources-pipeline-with-envs-converter', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-pipeline-with-envs-converter-error', - 'inputTopics': [ - 'resources-pipeline-with-envs-input-producer' - ], - 'outputTopic': 'resources-pipeline-with-envs-converter', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'example-output': { + 'configs': { + }, + 'type': 'output' + }, + 'resources-no-user-defined-components-streams-app-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + } +] + +snapshots['TestPipeline.test_pipelines_with_env_values test-pipeline'] = [ + { + 'app': { + 'commandLine': { + 'FAKE_ARG': 'override-arg' + }, + 'image': 'example-registry/fake-image', + 'imageTag': '0.0.1', + 'nameOverride': 'resources-pipeline-with-envs-input-producer', + 'schedule': '20 3/8 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-pipeline-with-envs-input-producer', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'input-producer', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-envs-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { + 'com/bakdata/kafka/fake': '1.0.0' }, - 'name': 'converter', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-envs-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-pipeline-with-envs-converter': { - 'configs': { - 'cleanup.policy': 'compact,delete', - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-pipeline-with-envs-converter-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 10, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-pipeline-with-envs-input-producer': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 12, + 'type': 'output', + 'value_schema': 'com.bakdata.fake.Produced' } + } + }, + 'type': 'scheduled-producer', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'converter-resources-pipeline-with-envs-converter', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 1, + 'minReplicas': 0, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + ] + }, + 'commandLine': { + 'CONVERT_XML': True + }, + 'nameOverride': 'resources-pipeline-with-envs-converter', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-pipeline-with-envs-converter-error', + 'inputTopics': [ + 'resources-pipeline-with-envs-input-producer' + ], + 'outputTopic': 'resources-pipeline-with-envs-converter', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'converter', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-envs-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'converter', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'filter-resources-pipeline-with-envs-filter', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 4, - 'minReplicas': 4, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - 'resources-pipeline-with-envs-filter' - ] - }, - 'commandLine': { - 'TYPE': 'nothing' - }, - 'image': 'fake-registry/filter', - 'imageTag': '2.4.1', - 'nameOverride': 'resources-pipeline-with-envs-filter', - 'replicaCount': 4, - 'resources': { - 'requests': { - 'memory': '3G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-pipeline-with-envs-filter-error', - 'inputTopics': [ - 'resources-pipeline-with-envs-converter' - ], - 'outputTopic': 'resources-pipeline-with-envs-filter', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-pipeline-with-envs-converter': { + 'configs': { + 'cleanup.policy': 'compact,delete', + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-pipeline-with-envs-converter-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 10, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'converter', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'filter-resources-pipeline-with-envs-filter', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 4, + 'minReplicas': 4, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + 'resources-pipeline-with-envs-filter' + ] + }, + 'commandLine': { + 'TYPE': 'nothing' + }, + 'image': 'fake-registry/filter', + 'imageTag': '2.4.1', + 'nameOverride': 'resources-pipeline-with-envs-filter', + 'replicaCount': 4, + 'resources': { + 'requests': { + 'memory': '3G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-pipeline-with-envs-filter-error', + 'inputTopics': [ + 'resources-pipeline-with-envs-converter' + ], + 'outputTopic': 'resources-pipeline-with-envs-filter', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'filter', + 'namespace': 'example-namespace', + 'prefix': 'resources-pipeline-with-envs-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'filter', - 'namespace': 'example-namespace', - 'prefix': 'resources-pipeline-with-envs-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-pipeline-with-envs-filter': { - 'configs': { - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-pipeline-with-envs-filter-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-pipeline-with-envs-filter': { + 'configs': { + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-pipeline-with-envs-filter-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'type': 'filter', - 'version': '2.4.2' - } - ] -} + } + }, + 'type': 'filter', + 'version': '2.4.2' + } +] -snapshots['TestPipeline.test_prefix_pipeline_component test-pipeline'] = { - 'components': [ - { - 'app': { - 'debug': True, - 'image': '${DOCKER_REGISTRY}/atm-demo-accountproducer', - 'imageTag': '1.0.0', - 'nameOverride': 'from-pipeline-component-account-producer', - 'prometheus': { - 'jmx': { - 'enabled': False - } - }, - 'replicaCount': 1, - 'schedule': '0 12 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'schemaRegistryUrl': 'http://localhost:8081/' - }, - 'suspend': True - }, - 'name': 'account-producer', - 'namespace': '${NAMESPACE}', - 'prefix': 'from-pipeline-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'type': 'producer-app', - 'version': '2.9.0' - } - ] -} +snapshots['TestPipeline.test_prefix_pipeline_component test-pipeline'] = [ + { + 'app': { + 'debug': True, + 'image': '${DOCKER_REGISTRY}/atm-demo-accountproducer', + 'imageTag': '1.0.0', + 'nameOverride': 'from-pipeline-component-account-producer', + 'prometheus': { + 'jmx': { + 'enabled': False + } + }, + 'replicaCount': 1, + 'schedule': '0 12 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'schemaRegistryUrl': 'http://localhost:8081/' + }, + 'suspend': True + }, + 'name': 'account-producer', + 'namespace': '${NAMESPACE}', + 'prefix': 'from-pipeline-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'type': 'producer-app', + 'version': '2.9.0' + } +] -snapshots['TestPipeline.test_read_from_component test-pipeline'] = { - 'components': [ - { - 'app': { - 'nameOverride': 'resources-read-from-component-producer1', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-read-from-component-producer1', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'producer1', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-producer1': { - 'configs': { - }, - 'type': 'output' - } - } - }, - 'type': 'producer-app', - 'version': '2.4.2' +snapshots['TestPipeline.test_read_from_component test-pipeline'] = [ + { + 'app': { + 'nameOverride': 'resources-read-from-component-producer1', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-read-from-component-producer1', + 'schemaRegistryUrl': 'http://localhost:8081/' + } }, - { - 'app': { - 'nameOverride': 'producer2', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-read-from-component-producer2', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'producer2', - 'namespace': 'example-namespace', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-producer2': { - 'configs': { - }, - 'type': 'output' - } - } - }, - 'type': 'producer-app', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'filter-resources-read-from-component-inflate-step', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 1, - 'minReplicas': 0, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - 'resources-read-from-component-inflate-step' - ] - }, - 'image': 'fake-registry/filter', - 'imageTag': '2.4.1', - 'nameOverride': 'resources-read-from-component-inflate-step', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-inflate-step-error', - 'inputTopics': [ - 'resources-read-from-component-producer2' - ], - 'outputTopic': 'resources-read-from-component-inflate-step', - 'schemaRegistryUrl': 'http://localhost:8081/' - } - }, - 'name': 'inflate-step', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-inflate-step': { - 'configs': { - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-read-from-component-inflate-step-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } - }, - 'type': 'should-inflate', - 'version': '2.4.2' - }, - { - 'app': { - 'batch.size': '2000', - 'behavior.on.malformed.documents': 'warn', - 'behavior.on.null.values': 'delete', - 'connection.compression': 'true', - 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', - 'key.ignore': 'false', - 'linger.ms': '5000', - 'max.buffered.records': '20000', - 'name': 'resources-read-from-component-inflate-step-inflated-sink-connector', - 'read.timeout.ms': '120000', - 'tasks.max': '1', - 'topics': 'resources-read-from-component-inflate-step', - 'transforms.changeTopic.replacement': 'resources-read-from-component-inflate-step-index-v1' - }, - 'name': 'inflate-step-inflated-sink-connector', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-kafka-connect-resetter', - 'url': 'https://bakdata.github.io/kafka-connect-resetter/' - }, - 'resetter_values': { - }, - 'to': { - 'models': { - }, - 'topics': { - 'inflate-step-inflated-sink-connector': { - 'configs': { - }, - 'role': 'test' - }, - 'kafka-sink-connector': { - 'configs': { - }, - 'type': 'output' - } - } - }, - 'type': 'kafka-sink-connector', - 'version': '1.0.4' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-inflate-step-inflated-streams-app', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-inflate-step-inflated-streams-app-error', - 'inputTopics': [ - 'kafka-sink-connector' - ], - 'outputTopic': 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app', - 'schemaRegistryUrl': 'http://localhost:8081/' - } + 'name': 'producer1', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'inflate-step-inflated-streams-app', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-read-from-component-inflate-step-inflated-streams-app-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-read-from-component-producer1': { + 'configs': { + }, + 'type': 'output' } + } + }, + 'type': 'producer-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'producer2', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-read-from-component-producer2', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'producer2', + 'namespace': 'example-namespace', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'filter-resources-read-from-component-inflate-step-without-prefix', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 1, - 'minReplicas': 0, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - 'resources-read-from-component-inflate-step-without-prefix' - ] - }, - 'image': 'fake-registry/filter', - 'imageTag': '2.4.1', - 'nameOverride': 'inflate-step-without-prefix', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-inflate-step-without-prefix-error', - 'inputTopics': [ - 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app' - ], - 'outputTopic': 'resources-read-from-component-inflate-step-without-prefix', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-read-from-component-producer2': { + 'configs': { + }, + 'type': 'output' } + } + }, + 'type': 'producer-app', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'filter-resources-read-from-component-inflate-step', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 1, + 'minReplicas': 0, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + 'resources-read-from-component-inflate-step' + ] + }, + 'image': 'fake-registry/filter', + 'imageTag': '2.4.1', + 'nameOverride': 'resources-read-from-component-inflate-step', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-inflate-step-error', + 'inputTopics': [ + 'resources-read-from-component-producer2' + ], + 'outputTopic': 'resources-read-from-component-inflate-step', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'inflate-step', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'inflate-step-without-prefix', - 'namespace': 'example-namespace', - 'prefix': '', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-inflate-step-without-prefix': { - 'configs': { - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-read-from-component-inflate-step-without-prefix-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-read-from-component-inflate-step': { + 'configs': { + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-read-from-component-inflate-step-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'should-inflate', + 'version': '2.4.2' + }, + { + 'app': { + 'batch.size': '2000', + 'behavior.on.malformed.documents': 'warn', + 'behavior.on.null.values': 'delete', + 'connection.compression': 'true', + 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', + 'key.ignore': 'false', + 'linger.ms': '5000', + 'max.buffered.records': '20000', + 'name': 'resources-read-from-component-inflate-step-inflated-sink-connector', + 'read.timeout.ms': '120000', + 'tasks.max': '1', + 'topics': 'resources-read-from-component-inflate-step', + 'transforms.changeTopic.replacement': 'resources-read-from-component-inflate-step-index-v1' + }, + 'name': 'inflate-step-inflated-sink-connector', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-kafka-connect-resetter', + 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + }, + 'resetter_values': { + }, + 'to': { + 'models': { }, - 'type': 'should-inflate', - 'version': '2.4.2' - }, - { - 'app': { - 'batch.size': '2000', - 'behavior.on.malformed.documents': 'warn', - 'behavior.on.null.values': 'delete', - 'connection.compression': 'true', - 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', - 'key.ignore': 'false', - 'linger.ms': '5000', - 'max.buffered.records': '20000', - 'name': 'resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector', - 'read.timeout.ms': '120000', - 'tasks.max': '1', - 'topics': 'resources-read-from-component-inflate-step-without-prefix', - 'transforms.changeTopic.replacement': 'resources-read-from-component-inflate-step-without-prefix-index-v1' - }, - 'name': 'inflate-step-without-prefix-inflated-sink-connector', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-kafka-connect-resetter', - 'url': 'https://bakdata.github.io/kafka-connect-resetter/' - }, - 'resetter_values': { - }, - 'to': { - 'models': { - }, - 'topics': { - 'inflate-step-without-prefix-inflated-sink-connector': { - 'configs': { - }, - 'role': 'test' - }, - 'kafka-sink-connector': { - 'configs': { - }, - 'type': 'output' - } + 'topics': { + 'inflate-step-inflated-sink-connector': { + 'configs': { + }, + 'role': 'test' + }, + 'kafka-sink-connector': { + 'configs': { + }, + 'type': 'output' } + } + }, + 'type': 'kafka-sink-connector', + 'version': '1.0.4' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-inflate-step-inflated-streams-app', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-inflate-step-inflated-streams-app-error', + 'inputTopics': [ + 'kafka-sink-connector' + ], + 'outputTopic': 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'inflate-step-inflated-streams-app', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'kafka-sink-connector', - 'version': '1.0.4' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app-error', - 'inputTopics': [ - 'kafka-sink-connector' - ], - 'outputTopic': 'inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app': { + 'configs': { + }, + 'type': 'output' + }, + 'resources-read-from-component-inflate-step-inflated-streams-app-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'filter-resources-read-from-component-inflate-step-without-prefix', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 1, + 'minReplicas': 0, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + 'resources-read-from-component-inflate-step-without-prefix' + ] + }, + 'image': 'fake-registry/filter', + 'imageTag': '2.4.1', + 'nameOverride': 'inflate-step-without-prefix', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-inflate-step-without-prefix-error', + 'inputTopics': [ + 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app' + ], + 'outputTopic': 'resources-read-from-component-inflate-step-without-prefix', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'inflate-step-without-prefix', + 'namespace': 'example-namespace', + 'prefix': '', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'inflate-step-without-prefix-inflated-streams-app', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-read-from-component-inflate-step-without-prefix': { + 'configs': { + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-read-from-component-inflate-step-without-prefix-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'should-inflate', + 'version': '2.4.2' + }, + { + 'app': { + 'batch.size': '2000', + 'behavior.on.malformed.documents': 'warn', + 'behavior.on.null.values': 'delete', + 'connection.compression': 'true', + 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', + 'key.ignore': 'false', + 'linger.ms': '5000', + 'max.buffered.records': '20000', + 'name': 'resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector', + 'read.timeout.ms': '120000', + 'tasks.max': '1', + 'topics': 'resources-read-from-component-inflate-step-without-prefix', + 'transforms.changeTopic.replacement': 'resources-read-from-component-inflate-step-without-prefix-index-v1' + }, + 'name': 'inflate-step-without-prefix-inflated-sink-connector', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-kafka-connect-resetter', + 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + }, + 'resetter_values': { + }, + 'to': { + 'models': { }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-consumer1', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-consumer1-error', - 'inputTopics': [ - 'resources-read-from-component-producer1' - ], - 'outputTopic': 'resources-read-from-component-consumer1', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'inflate-step-without-prefix-inflated-sink-connector': { + 'configs': { + }, + 'role': 'test' + }, + 'kafka-sink-connector': { + 'configs': { + }, + 'type': 'output' } + } + }, + 'type': 'kafka-sink-connector', + 'version': '1.0.4' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app-error', + 'inputTopics': [ + 'kafka-sink-connector' + ], + 'outputTopic': 'inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'inflate-step-without-prefix-inflated-streams-app', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'from': { - 'components': { - 'producer1': { - 'type': 'input' - } + 'topics': { + 'inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app': { + 'configs': { + }, + 'type': 'output' }, - 'topics': { + 'resources-read-from-component-inflate-step-without-prefix-inflated-streams-app-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'name': 'consumer1', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-consumer1': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-read-from-component-consumer1-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-consumer1', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-consumer1-error', + 'inputTopics': [ + 'resources-read-from-component-producer1' + ], + 'outputTopic': 'resources-read-from-component-consumer1', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { + 'producer1': { + 'type': 'input' } }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-consumer2', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-consumer2-error', - 'inputTopics': [ - 'resources-read-from-component-producer1', - 'resources-read-from-component-consumer1' - ], - 'schemaRegistryUrl': 'http://localhost:8081/' - } + 'topics': { + } + }, + 'name': 'consumer1', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'from': { - 'components': { - 'consumer1': { - 'type': 'input' + 'topics': { + 'resources-read-from-component-consumer1': { + 'configs': { }, - 'producer1': { - 'type': 'input' - } + 'type': 'output' }, - 'topics': { - } - }, - 'name': 'consumer2', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-consumer2-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } - }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-consumer3', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-consumer3-error', - 'inputTopics': [ - 'resources-read-from-component-producer1', - 'resources-read-from-component-producer2' - ], - 'schemaRegistryUrl': 'http://localhost:8081/' + 'resources-read-from-component-consumer1-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'from': { - 'components': { - 'producer2': { - 'type': 'input' - } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-consumer2', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-consumer2-error', + 'inputTopics': [ + 'resources-read-from-component-producer1', + 'resources-read-from-component-consumer1' + ], + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { + 'consumer1': { + 'type': 'input' }, - 'topics': { - 'resources-read-from-component-producer1': { - 'type': 'input' - } + 'producer1': { + 'type': 'input' } }, - 'name': 'consumer3', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-consumer3-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } + 'topics': { + } + }, + 'name': 'consumer2', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-consumer4', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-consumer4-error', - 'inputTopics': [ - 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app' - ], - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-read-from-component-consumer2-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'from': { - 'components': { - 'inflate-step': { - 'type': 'input' - } - }, - 'topics': { + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-consumer3', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-consumer3-error', + 'inputTopics': [ + 'resources-read-from-component-producer1', + 'resources-read-from-component-producer2' + ], + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { + 'producer2': { + 'type': 'input' } }, - 'name': 'consumer4', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-consumer4-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-read-from-component-producer1': { + 'type': 'input' } + } + }, + 'name': 'consumer3', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'streams-app', - 'version': '2.4.2' - }, - { - 'app': { - 'nameOverride': 'resources-read-from-component-consumer5', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-read-from-component-consumer5-error', - 'inputTopics': [ - 'inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app' - ], - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-read-from-component-consumer3-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'from': { - 'components': { - 'inflate-step-without-prefix': { - 'type': 'input' - } - }, - 'topics': { + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-consumer4', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-consumer4-error', + 'inputTopics': [ + 'resources-read-from-component-inflate-step-inflate-step-inflated-streams-app' + ], + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { + 'inflate-step': { + 'type': 'input' } }, - 'name': 'consumer5', - 'namespace': 'example-namespace', - 'prefix': 'resources-read-from-component-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-read-from-component-consumer5-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } - } + 'topics': { + } + }, + 'name': 'consumer4', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'streams-app', - 'version': '2.4.2' - } - ] -} - -snapshots['TestPipeline.test_substitute_in_component test-pipeline'] = { - 'components': [ - { - 'app': { - 'commandLine': { - 'FAKE_ARG': 'fake-arg-value' - }, - 'image': 'example-registry/fake-image', - 'imageTag': '0.0.1', - 'labels': { - 'app_name': 'scheduled-producer', - 'app_schedule': '30 3/8 * * *', - 'app_type': 'scheduled-producer' - }, - 'nameOverride': 'resources-component-type-substitution-scheduled-producer', - 'schedule': '30 3/8 * * *', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'resources-component-type-substitution-scheduled-producer', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-read-from-component-consumer4-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'name': 'scheduled-producer', - 'namespace': 'example-namespace', - 'prefix': 'resources-component-type-substitution-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - 'com/bakdata/kafka/fake': '1.0.0' - }, - 'topics': { - 'resources-component-type-substitution-scheduled-producer': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 12, - 'type': 'output', - 'value_schema': 'com.bakdata.fake.Produced' - } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + }, + { + 'app': { + 'nameOverride': 'resources-read-from-component-consumer5', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-read-from-component-consumer5-error', + 'inputTopics': [ + 'inflate-step-without-prefix-inflate-step-without-prefix-inflated-streams-app' + ], + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { + 'inflate-step-without-prefix': { + 'type': 'input' } }, - 'type': 'scheduled-producer', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'converter-resources-component-type-substitution-converter', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 1, - 'minReplicas': 0, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - ] - }, - 'commandLine': { - 'CONVERT_XML': True - }, - 'nameOverride': 'resources-component-type-substitution-converter', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-component-type-substitution-converter-error', - 'inputTopics': [ - 'resources-component-type-substitution-scheduled-producer' - ], - 'outputTopic': 'resources-component-type-substitution-converter', - 'schemaRegistryUrl': 'http://localhost:8081/' - } + 'topics': { + } + }, + 'name': 'consumer5', + 'namespace': 'example-namespace', + 'prefix': 'resources-read-from-component-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'converter', - 'namespace': 'example-namespace', - 'prefix': 'resources-component-type-substitution-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-component-type-substitution-converter': { - 'configs': { - 'cleanup.policy': 'compact,delete', - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-component-type-substitution-converter-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 10, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-read-from-component-consumer5-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'streams-app', + 'version': '2.4.2' + } +] + +snapshots['TestPipeline.test_substitute_in_component test-pipeline'] = [ + { + 'app': { + 'commandLine': { + 'FAKE_ARG': 'fake-arg-value' + }, + 'image': 'example-registry/fake-image', + 'imageTag': '0.0.1', + 'labels': { + 'app_name': 'scheduled-producer', + 'app_schedule': '30 3/8 * * *', + 'app_type': 'scheduled-producer' + }, + 'nameOverride': 'resources-component-type-substitution-scheduled-producer', + 'schedule': '30 3/8 * * *', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'resources-component-type-substitution-scheduled-producer', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'scheduled-producer', + 'namespace': 'example-namespace', + 'prefix': 'resources-component-type-substitution-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { + 'com/bakdata/kafka/fake': '1.0.0' }, - 'type': 'converter', - 'version': '2.4.2' - }, - { - 'app': { - 'autoscaling': { - 'consumerGroup': 'filter-resources-component-type-substitution-filter-app', - 'cooldownPeriod': 300, - 'enabled': True, - 'lagThreshold': 10000, - 'maxReplicas': 4, - 'minReplicas': 4, - 'offsetResetPolicy': 'earliest', - 'pollingInterval': 30, - 'topics': [ - 'resources-component-type-substitution-filter-app' - ] - }, - 'commandLine': { - 'TYPE': 'nothing' - }, - 'image': 'fake-registry/filter', - 'imageTag': '2.4.1', - 'labels': { - 'app_name': 'filter-app', - 'app_resources_requests_memory': '3G', - 'app_type': 'filter', - 'filter': 'filter-app-filter', - 'test_placeholder_in_placeholder': 'filter-app-filter' - }, - 'nameOverride': 'resources-component-type-substitution-filter-app', - 'replicaCount': 4, - 'resources': { - 'requests': { - 'memory': '3G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-component-type-substitution-filter-app-error', - 'inputTopics': [ - 'resources-component-type-substitution-converter' - ], - 'outputTopic': 'resources-component-type-substitution-filter-app', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'resources-component-type-substitution-scheduled-producer': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 12, + 'type': 'output', + 'value_schema': 'com.bakdata.fake.Produced' } + } + }, + 'type': 'scheduled-producer', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'converter-resources-component-type-substitution-converter', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 1, + 'minReplicas': 0, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + ] + }, + 'commandLine': { + 'CONVERT_XML': True + }, + 'nameOverride': 'resources-component-type-substitution-converter', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-component-type-substitution-converter-error', + 'inputTopics': [ + 'resources-component-type-substitution-scheduled-producer' + ], + 'outputTopic': 'resources-component-type-substitution-converter', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'converter', + 'namespace': 'example-namespace', + 'prefix': 'resources-component-type-substitution-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'filter-app', - 'namespace': 'example-namespace', - 'prefix': 'resources-component-type-substitution-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'resources-component-type-substitution-filter-app': { - 'configs': { - 'retention.ms': '-1' - }, - 'partitions_count': 50, - 'type': 'output' - }, - 'resources-component-type-substitution-filter-app-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'resources-component-type-substitution-converter': { + 'configs': { + 'cleanup.policy': 'compact,delete', + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-component-type-substitution-converter-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 10, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } + } + }, + 'type': 'converter', + 'version': '2.4.2' + }, + { + 'app': { + 'autoscaling': { + 'consumerGroup': 'filter-resources-component-type-substitution-filter-app', + 'cooldownPeriod': 300, + 'enabled': True, + 'lagThreshold': 10000, + 'maxReplicas': 4, + 'minReplicas': 4, + 'offsetResetPolicy': 'earliest', + 'pollingInterval': 30, + 'topics': [ + 'resources-component-type-substitution-filter-app' + ] + }, + 'commandLine': { + 'TYPE': 'nothing' + }, + 'image': 'fake-registry/filter', + 'imageTag': '2.4.1', + 'labels': { + 'app_name': 'filter-app', + 'app_resources_requests_memory': '3G', + 'app_type': 'filter', + 'filter': 'filter-app-filter', + 'test_placeholder_in_placeholder': 'filter-app-filter' + }, + 'nameOverride': 'resources-component-type-substitution-filter-app', + 'replicaCount': 4, + 'resources': { + 'requests': { + 'memory': '3G' + } + }, + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-component-type-substitution-filter-app-error', + 'inputTopics': [ + 'resources-component-type-substitution-converter' + ], + 'outputTopic': 'resources-component-type-substitution-filter-app', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'filter-app', + 'namespace': 'example-namespace', + 'prefix': 'resources-component-type-substitution-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'filter', - 'version': '2.4.2' - } - ] -} + 'topics': { + 'resources-component-type-substitution-filter-app': { + 'configs': { + 'retention.ms': '-1' + }, + 'partitions_count': 50, + 'type': 'output' + }, + 'resources-component-type-substitution-filter-app-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' + } + } + }, + 'type': 'filter', + 'version': '2.4.2' + } +] -snapshots['TestPipeline.test_with_custom_config_with_absolute_defaults_path test-pipeline'] = { - 'components': [ - { - 'app': { - 'nameOverride': 'resources-custom-config-app1', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'app1-test-topic', - 'schemaRegistryUrl': 'http://localhost:8081/' +snapshots['TestPipeline.test_with_custom_config_with_absolute_defaults_path test-pipeline'] = [ + { + 'app': { + 'nameOverride': 'resources-custom-config-app1', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' } }, - 'name': 'app1', - 'namespace': 'development-namespace', - 'prefix': 'resources-custom-config-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'app1-test-topic': { - 'configs': { - }, - 'partitions_count': 3, - 'type': 'output' - } - } + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'app1-test-topic', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app1', + 'namespace': 'development-namespace', + 'prefix': 'resources-custom-config-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'producer-app', - 'version': '2.9.0' - }, - { - 'app': { - 'image': 'some-image', - 'labels': { - 'pipeline': 'resources-custom-config' - }, - 'nameOverride': 'resources-custom-config-app2', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'errorTopic': 'app2-dead-letter-topic', - 'inputTopics': [ - 'app1-test-topic' - ], - 'outputTopic': 'app2-test-topic', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'app1-test-topic': { + 'configs': { + }, + 'partitions_count': 3, + 'type': 'output' } + } + }, + 'type': 'producer-app', + 'version': '2.9.0' + }, + { + 'app': { + 'image': 'some-image', + 'labels': { + 'pipeline': 'resources-custom-config' + }, + 'nameOverride': 'resources-custom-config-app2', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'errorTopic': 'app2-dead-letter-topic', + 'inputTopics': [ + 'app1-test-topic' + ], + 'outputTopic': 'app2-test-topic', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app2', + 'namespace': 'development-namespace', + 'prefix': 'resources-custom-config-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'app2', - 'namespace': 'development-namespace', - 'prefix': 'resources-custom-config-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'app2-dead-letter-topic': { - 'configs': { - }, - 'partitions_count': 1, - 'type': 'error' - }, - 'app2-test-topic': { - 'configs': { - }, - 'partitions_count': 3, - 'type': 'output' - } + 'topics': { + 'app2-dead-letter-topic': { + 'configs': { + }, + 'partitions_count': 1, + 'type': 'error' + }, + 'app2-test-topic': { + 'configs': { + }, + 'partitions_count': 3, + 'type': 'output' } - }, - 'type': 'streams-app', - 'version': '2.9.0' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + } +] -snapshots['TestPipeline.test_with_custom_config_with_relative_defaults_path test-pipeline'] = { - 'components': [ - { - 'app': { - 'nameOverride': 'resources-custom-config-app1', - 'resources': { - 'limits': { - 'memory': '2G' - }, - 'requests': { - 'memory': '2G' - } - }, - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'extraOutputTopics': { - }, - 'outputTopic': 'app1-test-topic', - 'schemaRegistryUrl': 'http://localhost:8081/' +snapshots['TestPipeline.test_with_custom_config_with_relative_defaults_path test-pipeline'] = [ + { + 'app': { + 'nameOverride': 'resources-custom-config-app1', + 'resources': { + 'limits': { + 'memory': '2G' + }, + 'requests': { + 'memory': '2G' } }, - 'name': 'app1', - 'namespace': 'development-namespace', - 'prefix': 'resources-custom-config-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'app1-test-topic': { - 'configs': { - }, - 'partitions_count': 3, - 'type': 'output' - } - } + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'extraOutputTopics': { + }, + 'outputTopic': 'app1-test-topic', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app1', + 'namespace': 'development-namespace', + 'prefix': 'resources-custom-config-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'type': 'producer-app', - 'version': '2.9.0' - }, - { - 'app': { - 'image': 'some-image', - 'labels': { - 'pipeline': 'resources-custom-config' - }, - 'nameOverride': 'resources-custom-config-app2', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'errorTopic': 'app2-dead-letter-topic', - 'inputTopics': [ - 'app1-test-topic' - ], - 'outputTopic': 'app2-test-topic', - 'schemaRegistryUrl': 'http://localhost:8081/' + 'topics': { + 'app1-test-topic': { + 'configs': { + }, + 'partitions_count': 3, + 'type': 'output' } + } + }, + 'type': 'producer-app', + 'version': '2.9.0' + }, + { + 'app': { + 'image': 'some-image', + 'labels': { + 'pipeline': 'resources-custom-config' + }, + 'nameOverride': 'resources-custom-config-app2', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'errorTopic': 'app2-dead-letter-topic', + 'inputTopics': [ + 'app1-test-topic' + ], + 'outputTopic': 'app2-test-topic', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'name': 'app2', + 'namespace': 'development-namespace', + 'prefix': 'resources-custom-config-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'app2', - 'namespace': 'development-namespace', - 'prefix': 'resources-custom-config-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'app2-dead-letter-topic': { - 'configs': { - }, - 'partitions_count': 1, - 'type': 'error' - }, - 'app2-test-topic': { - 'configs': { - }, - 'partitions_count': 3, - 'type': 'output' - } + 'topics': { + 'app2-dead-letter-topic': { + 'configs': { + }, + 'partitions_count': 1, + 'type': 'error' + }, + 'app2-test-topic': { + 'configs': { + }, + 'partitions_count': 3, + 'type': 'output' } - }, - 'type': 'streams-app', - 'version': '2.9.0' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + } +] -snapshots['TestPipeline.test_with_env_defaults test-pipeline'] = { - 'components': [ - { - 'app': { - 'image': 'fake-image', - 'nameOverride': 'resources-kafka-connect-sink-streams-app-development', - 'streams': { - 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', - 'config': { - 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' - }, - 'errorTopic': 'resources-kafka-connect-sink-streams-app-development-error', - 'inputTopics': [ - 'example-topic' - ], - 'outputTopic': 'example-output', - 'schemaRegistryUrl': 'http://localhost:8081/' - } +snapshots['TestPipeline.test_with_env_defaults test-pipeline'] = [ + { + 'app': { + 'image': 'fake-image', + 'nameOverride': 'resources-kafka-connect-sink-streams-app-development', + 'streams': { + 'brokers': 'http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092', + 'config': { + 'large.message.id.generator': 'com.bakdata.kafka.MurmurHashIdGenerator' + }, + 'errorTopic': 'resources-kafka-connect-sink-streams-app-development-error', + 'inputTopics': [ + 'example-topic' + ], + 'outputTopic': 'example-output', + 'schemaRegistryUrl': 'http://localhost:8081/' + } + }, + 'from': { + 'components': { }, - 'from': { - 'components': { - }, - 'topics': { - 'example-topic': { - 'type': 'input' - } + 'topics': { + 'example-topic': { + 'type': 'input' } + } + }, + 'name': 'streams-app-development', + 'namespace': 'development-namespace', + 'prefix': 'resources-kafka-connect-sink-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-streams-bootstrap', + 'url': 'https://bakdata.github.io/streams-bootstrap/' + }, + 'to': { + 'models': { }, - 'name': 'streams-app-development', - 'namespace': 'development-namespace', - 'prefix': 'resources-kafka-connect-sink-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-streams-bootstrap', - 'url': 'https://bakdata.github.io/streams-bootstrap/' - }, - 'to': { - 'models': { - }, - 'topics': { - 'example-output': { - 'configs': { - }, - 'type': 'output' - }, - 'resources-kafka-connect-sink-streams-app-development-error': { - 'configs': { - 'cleanup.policy': 'compact,delete' - }, - 'partitions_count': 1, - 'type': 'error', - 'value_schema': 'com.bakdata.kafka.DeadLetter' - } + 'topics': { + 'example-output': { + 'configs': { + }, + 'type': 'output' + }, + 'resources-kafka-connect-sink-streams-app-development-error': { + 'configs': { + 'cleanup.policy': 'compact,delete' + }, + 'partitions_count': 1, + 'type': 'error', + 'value_schema': 'com.bakdata.kafka.DeadLetter' } - }, - 'type': 'streams-app', - 'version': '2.9.0' - }, - { - 'app': { - 'batch.size': '2000', - 'behavior.on.malformed.documents': 'warn', - 'behavior.on.null.values': 'delete', - 'connection.compression': 'true', - 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', - 'key.ignore': 'false', - 'linger.ms': '5000', - 'max.buffered.records': '20000', - 'name': 'resources-kafka-connect-sink-es-sink-connector', - 'read.timeout.ms': '120000', - 'tasks.max': '1', - 'topics': 'example-output' - }, - 'name': 'es-sink-connector', - 'namespace': 'example-namespace', - 'prefix': 'resources-kafka-connect-sink-', - 'repo_config': { - 'repo_auth_flags': { - 'insecure_skip_tls_verify': False - }, - 'repository_name': 'bakdata-kafka-connect-resetter', - 'url': 'https://bakdata.github.io/kafka-connect-resetter/' - }, - 'resetter_values': { - }, - 'type': 'kafka-sink-connector', - 'version': '1.0.4' - } - ] -} + } + }, + 'type': 'streams-app', + 'version': '2.9.0' + }, + { + 'app': { + 'batch.size': '2000', + 'behavior.on.malformed.documents': 'warn', + 'behavior.on.null.values': 'delete', + 'connection.compression': 'true', + 'connector.class': 'io.confluent.connect.elasticsearch.ElasticsearchSinkConnector', + 'key.ignore': 'false', + 'linger.ms': '5000', + 'max.buffered.records': '20000', + 'name': 'resources-kafka-connect-sink-es-sink-connector', + 'read.timeout.ms': '120000', + 'tasks.max': '1', + 'topics': 'example-output' + }, + 'name': 'es-sink-connector', + 'namespace': 'example-namespace', + 'prefix': 'resources-kafka-connect-sink-', + 'repo_config': { + 'repo_auth_flags': { + 'insecure_skip_tls_verify': False + }, + 'repository_name': 'bakdata-kafka-connect-resetter', + 'url': 'https://bakdata.github.io/kafka-connect-resetter/' + }, + 'resetter_values': { + }, + 'type': 'kafka-sink-connector', + 'version': '1.0.4' + } +] diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index 4a43ce9d5..9be8edf7c 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -8,7 +8,7 @@ import kpops from kpops.cli.main import app -from kpops.pipeline_generator.pipeline import ParsingException, ValidationError +from kpops.pipeline import ParsingException, ValidationError runner = CliRunner() @@ -97,8 +97,8 @@ def test_name_equal_prefix_name_concatenation(self): enriched_pipeline: dict = yaml.safe_load(result.stdout) - assert enriched_pipeline["components"][0]["prefix"] == "my-fake-prefix-" - assert enriched_pipeline["components"][0]["name"] == "my-streams-app" + assert enriched_pipeline[0]["prefix"] == "my-fake-prefix-" + assert enriched_pipeline[0]["name"] == "my-streams-app" def test_pipelines_with_env_values(self, snapshot: SnapshotTest): result = runner.invoke( @@ -161,33 +161,28 @@ def test_substitute_in_component(self, snapshot: SnapshotTest): enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][0]["prefix"] - == "resources-component-type-substitution-" + enriched_pipeline[0]["prefix"] == "resources-component-type-substitution-" ) - assert enriched_pipeline["components"][0]["name"] == "scheduled-producer" + assert enriched_pipeline[0]["name"] == "scheduled-producer" - labels = enriched_pipeline["components"][0]["app"]["labels"] + labels = enriched_pipeline[0]["app"]["labels"] assert labels["app_name"] == "scheduled-producer" assert labels["app_type"] == "scheduled-producer" assert labels["app_schedule"] == "30 3/8 * * *" assert ( - enriched_pipeline["components"][2]["app"]["labels"][ - "app_resources_requests_memory" - ] + enriched_pipeline[2]["app"]["labels"]["app_resources_requests_memory"] == "3G" ) assert ( "resources-component-type-substitution-scheduled-producer" - in enriched_pipeline["components"][0]["to"]["topics"] + in enriched_pipeline[0]["to"]["topics"] ) assert ( "resources-component-type-substitution-converter-error" - in enriched_pipeline["components"][1]["to"]["topics"] + in enriched_pipeline[1]["to"]["topics"] ) assert ( - enriched_pipeline["components"][2]["app"]["labels"][ - "test_placeholder_in_placeholder" - ] + enriched_pipeline[2]["app"]["labels"]["test_placeholder_in_placeholder"] == "filter-app-filter" ) @@ -229,7 +224,7 @@ def test_kafka_connector_config_parsing(self): catch_exceptions=False, ) enriched_pipeline: dict = yaml.safe_load(result.stdout) - sink_connector = enriched_pipeline["components"][0] + sink_connector = enriched_pipeline[0] assert ( sink_connector["app"]["errors.deadletterqueue.topic.name"] == "kafka-sink-connector-error-topic" @@ -379,11 +374,11 @@ def test_with_custom_config_with_relative_defaults_path( assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) - producer_details = enriched_pipeline["components"][0] + producer_details = enriched_pipeline[0] output_topic = producer_details["app"]["streams"]["outputTopic"] assert output_topic == "app1-test-topic" - streams_app_details = enriched_pipeline["components"][1] + streams_app_details = enriched_pipeline[1] output_topic = streams_app_details["app"]["streams"]["outputTopic"] assert output_topic == "app2-test-topic" error_topic = streams_app_details["app"]["streams"]["errorTopic"] @@ -424,11 +419,11 @@ def test_with_custom_config_with_absolute_defaults_path( assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) - producer_details = enriched_pipeline["components"][0] + producer_details = enriched_pipeline[0] output_topic = producer_details["app"]["streams"]["outputTopic"] assert output_topic == "app1-test-topic" - streams_app_details = enriched_pipeline["components"][1] + streams_app_details = enriched_pipeline[1] output_topic = streams_app_details["app"]["streams"]["outputTopic"] assert output_topic == "app2-test-topic" error_topic = streams_app_details["app"]["streams"]["errorTopic"] @@ -457,11 +452,11 @@ def test_default_config(self, snapshot: SnapshotTest): assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) - producer_details = enriched_pipeline["components"][0] + producer_details = enriched_pipeline[0] output_topic = producer_details["app"]["streams"]["outputTopic"] assert output_topic == "resources-custom-config-app1" - streams_app_details = enriched_pipeline["components"][1] + streams_app_details = enriched_pipeline[1] output_topic = streams_app_details["app"]["streams"]["outputTopic"] assert output_topic == "resources-custom-config-app2" error_topic = streams_app_details["app"]["streams"]["errorTopic"] @@ -488,10 +483,7 @@ def test_env_vars_precedence_over_config(self, monkeypatch: pytest.MonkeyPatch): ) assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) - assert ( - enriched_pipeline["components"][0]["app"]["streams"]["brokers"] - == "env_broker" - ) + assert enriched_pipeline[0]["app"]["streams"]["brokers"] == "env_broker" def test_nested_config_env_vars(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv( @@ -515,7 +507,7 @@ def test_nested_config_env_vars(self, monkeypatch: pytest.MonkeyPatch): assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][0]["app"]["streams"]["schemaRegistryUrl"] + enriched_pipeline[0]["app"]["streams"]["schemaRegistryUrl"] == "http://somename:1234/" ) @@ -541,7 +533,7 @@ def test_env_specific_config_env_def_in_env_var( assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][0]["app"]["streams"]["schemaRegistryUrl"] + enriched_pipeline[0]["app"]["streams"]["schemaRegistryUrl"] == "http://production:8081/" ) @@ -579,8 +571,7 @@ def test_env_specific_config_env_def_in_cli( assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][0]["app"]["streams"]["schemaRegistryUrl"] - == expected_url + enriched_pipeline[0]["app"]["streams"]["schemaRegistryUrl"] == expected_url ) def test_config_dir_doesnt_exist(self): @@ -645,7 +636,7 @@ def test_dotenv_support(self): enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][1]["app"]["streams"]["schemaRegistryUrl"] + enriched_pipeline[1]["app"]["streams"]["schemaRegistryUrl"] == "http://notlocalhost:8081/" ) @@ -667,9 +658,9 @@ def test_short_topic_definition(self): enriched_pipeline: dict = yaml.safe_load(result.stdout) - output_topics = enriched_pipeline["components"][4]["to"]["topics"] - input_topics = enriched_pipeline["components"][4]["from"]["topics"] - input_components = enriched_pipeline["components"][4]["from"]["components"] + output_topics = enriched_pipeline[4]["to"]["topics"] + input_topics = enriched_pipeline[4]["from"]["topics"] + input_components = enriched_pipeline[4]["from"]["components"] assert "type" not in output_topics["output-topic"] assert output_topics["error-topic"]["type"] == "error" assert "type" not in output_topics["extra-topic"] @@ -748,6 +739,6 @@ def test_temp_trim_release_name(self): assert result.exit_code == 0 enriched_pipeline: dict = yaml.safe_load(result.stdout) assert ( - enriched_pipeline["components"][0]["name"] + enriched_pipeline[0]["name"] == "in-order-to-have-len-fifty-two-name-should-end--here" )