Support networkx to represent components as a graph

docs/docs/resources/examples/defaults.md

@@ -9,11 +9,18 @@
         --8<--
     ```
 
-## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops/defaults){target=_blank}
+<!-- dprint-ignore-end -->
+
+## [Word-count Pipeline](https://github.com/bakdata/kpops-examples/tree/main/word-count/deployment/kpops){target=_blank}
+
+<!-- dprint-ignore-start -->
 
 ??? example "defaults.yaml"
     ```yaml
-        --8<--
-        https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/defaults/defaults.yaml
-        --8<--
+    --8<--
+    https://raw.githubusercontent.com/bakdata/kpops-examples/main/word-count/deployment/kpops/defaults.yaml
+    --8<--
     ```
+
+<!-- dprint-ignore-end -->
+

kpops/components/base_components/kafka_connector.py

@@ -411,6 +411,11 @@ class KafkaSinkConnector(KafkaConnector):
         exclude=True,
     )
 
+    @override
+    def get_input_topics(self) -> list[str]:
+        topics = getattr(self.app, "topics", None)
+        return topics.split(",") if topics is not None else []
+
     @override
     def add_input_topics(self, topics: list[str]) -> None:
         existing_topics: str | None = getattr(self.app, "topics", None)

kpops/components/base_components/pipeline_component.py

@@ -75,6 +75,24 @@ def __init__(self, **kwargs) -> None:
         self.set_input_topics()
         self.set_output_topics()
 
+    def get_input_topics(self) -> list[str] | None:
+        """
+        Get all the input topics from config.
+        """
+
+    def get_extra_output_topics(self) -> dict[str, str] | None:
+        """
+        Get extra output topics list from config.
+        """
+
+    def get_extra_input_topics(self) -> dict[str, list[str]] | None:
+        """
+        Get extra input topics list from config.
+        """
+
+    def get_output_topic(self) -> str | None:
+        ...
+
     def add_input_topics(self, topics: list[str]) -> None:
         """Add given topics to the list of input topics.
 

kpops/components/streams_bootstrap/producer/producer_app.py

@@ -63,6 +63,14 @@ def apply_to_outputs(self, name: str, topic: TopicConfig) -> None:
             case _:
                 super().apply_to_outputs(name, topic)
 
+    @override
+    def get_output_topic(self) -> str | None:
+        return self.app.streams.output_topic
+
+    @override
+    def get_extra_output_topics(self) -> dict[str, str]:
+        return self.app.streams.extra_output_topics
+
     @override
     def set_output_topic(self, topic_name: str) -> None:
         self.app.streams.output_topic = topic_name

kpops/components/streams_bootstrap/streams/streams_app.py

@@ -42,6 +42,18 @@ class StreamsApp(KafkaApp):
     class Config(DescConfig):
         extra = Extra.allow
 
+    @override
+    def get_input_topics(self) -> list[str] | None:
+        return self.app.streams.input_topics
+
+    @override
+    def get_extra_input_topics(self) -> dict[str, list[str]] | None:
+        return self.app.streams.extra_input_topics
+
+    @override
+    def get_output_topic(self) -> str | None:
+        return self.app.streams.output_topic
+
     @override
     def add_input_topics(self, topics: list[str]) -> None:
         self.app.streams.add_input_topics(topics)

kpops/pipeline_generator/pipeline.py

@@ -5,10 +5,12 @@
 from collections import Counter
 from collections.abc import Iterator
 from contextlib import suppress
+from itertools import chain
 from pathlib import Path
 
+import networkx as nx
 import yaml
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from rich.console import Console
 from rich.syntax import Syntax
 
@@ -35,6 +37,10 @@ class PipelineComponents(BaseModel):
     """Stores the pipeline components"""
 
     components: list[PipelineComponent] = []
+    graph: nx.DiGraph = Field(default=nx.DiGraph(), exclude=True)
+
+    class Config:
+        arbitrary_types_allowed = True
 
     @property
     def last(self) -> PipelineComponent:
@@ -59,6 +65,10 @@ def __iter__(self) -> Iterator[PipelineComponent]:
     def __len__(self) -> int:
         return len(self.components)
 
+    def validate_graph_components(self) -> None:
+        if not nx.is_directed_acyclic_graph(self.graph):
+            raise ValueError("Pipeline contains cycles.")
+
     def validate_unique_names(self) -> None:
         step_names = [component.name for component in self.components]
         duplicates = [name for name, count in Counter(step_names).items() if count > 1]
@@ -67,6 +77,55 @@ def validate_unique_names(self) -> None:
                 f"step names should be unique. duplicate step names: {', '.join(duplicates)}"
             )
 
+    def generate_graph(self) -> None:
+        for component in self.components:
+            all_input_topics = self.__get_all_input_topics(component)
+            all_output_topics = self.__get_all_output_topics(component)
+
+            component_node_name = self.__get_vertex_component_name(component)
+            self.graph.add_node(component_node_name)
+
+            self.__add_ingoing_edges(all_input_topics, component_node_name)
+            self.__add_outgoing_edges(all_output_topics, component_node_name)
+
+    def __add_outgoing_edges(
+        self, all_output_topics: list[str], component_node_name: str
+    ) -> None:
+        for output_topic in all_output_topics:
+            self.graph.add_node(output_topic)
+            self.graph.add_edge(component_node_name, output_topic)
+
+    def __add_ingoing_edges(
+        self, all_input_topics: list[str], component_node_name: str
+    ) -> None:
+        for input_topic in all_input_topics:
+            self.graph.add_node(input_topic)
+            self.graph.add_edge(input_topic, component_node_name)
+
+    @staticmethod
+    def __get_vertex_component_name(component: PipelineComponent) -> str:
+        return f"component-{component.name}"
+
+    def __get_all_output_topics(self, component: PipelineComponent) -> list[str]:
+        all_output_topics: list[str] = []
+        output_topic = component.get_output_topic()
+        extra_output_topics = component.get_extra_output_topics()
+        if output_topic is not None:
+            all_output_topics.append(output_topic)
+        if extra_output_topics:
+            all_output_topics.extend(list(extra_output_topics.values()))
+        return all_output_topics
+
+    def __get_all_input_topics(self, component: PipelineComponent) -> list[str]:
+        input_topics = component.get_input_topics()
+        extra_input_topics = component.get_extra_input_topics()
+        all_input_topics: list[str] = []
+        if input_topics is not None:
+            all_input_topics.extend(input_topics)
+        if extra_input_topics:
+            all_input_topics.extend(chain(*extra_input_topics.values()))
+        return all_input_topics
+
     @staticmethod
     def _populate_component_name(component: PipelineComponent) -> None:
         component.name = component.prefix + component.name
@@ -112,6 +171,7 @@ def __init__(
         self.registry = registry
         self.env_components_index = create_env_components_index(environment_components)
         self.parse_components(component_list)
+        self.components.generate_graph()
         self.validate()
 
     @classmethod
@@ -327,6 +387,7 @@ def substitute_in_component(self, component_as_dict: dict) -> dict:
 
     def validate(self) -> None:
         self.components.validate_unique_names()
+        self.components.validate_graph_components()
 
     @staticmethod
     def pipeline_filename_environment(path: Path, config: PipelineConfig) -> Path:

pyproject.toml

@@ -38,6 +38,7 @@ cachetools = "^5.2.0"
 dictdiffer = "^0.9.0"
 python-schema-registry-client = "^2.4.1"
 httpx = "^0.24.1"
+networkx = "^3.1"
 
 
 [tool.poetry.group.dev.dependencies]

tests/components/test_producer_app.py

@@ -240,3 +240,37 @@ async def test_should_clean_producer_app_and_deploy_clean_up_job_and_delete_clea
                 "test-namespace", self.PRODUCER_APP_CLEAN_NAME, False
             ),
         ]
+
+    def test_get_output_topics(
+        self, config: PipelineConfig, handlers: ComponentHandlers
+    ):
+        producer_app = ProducerApp(
+            name=self.PRODUCER_APP_NAME,
+            config=config,
+            handlers=handlers,
+            **{
+                "namespace": "test-namespace",
+                "app": {
+                    "namespace": "test-namespace",
+                    "streams": {"brokers": "fake-broker:9092"},
+                },
+                "to": {
+                    "topics": {
+                        "${output_topic_name}": TopicConfig(
+                            type=OutputTopicTypes.OUTPUT, partitions_count=10
+                        ),
+                        "extra-topic-1": TopicConfig(
+                            type=OutputTopicTypes.EXTRA,
+                            role="first-extra-topic",
+                            partitions_count=10,
+                        ),
+                    }
+                },
+            },
+        )
+
+        assert producer_app.get_output_topic() == "${output_topic_name}"
+        assert producer_app.get_extra_output_topics() == {
+            "first-extra-topic": "extra-topic-1"
+        }
+        assert producer_app.get_input_topics() is None

tests/components/test_streams_app.py

@@ -429,3 +429,62 @@ async def test_should_clean_streams_app_and_deploy_clean_up_job_and_delete_clean
                 "test-namespace", self.STREAMS_APP_CLEAN_NAME, dry_run
             ),
         ]
+
+    @pytest.mark.asyncio
+    async def test_get_topics(
+        self, config: PipelineConfig, handlers: ComponentHandlers
+    ):
+        streams_app = StreamsApp(
+            name=self.STREAMS_APP_NAME,
+            config=config,
+            handlers=handlers,
+            **{
+                "namespace": "test-namespace",
+                "app": {
+                    "streams": {"brokers": "fake-broker:9092"},
+                },
+                "from": {
+                    "topics": {
+                        "example-input": {"type": "input"},
+                        "b": {"type": "input"},
+                        "a": {"type": "input"},
+                        "topic-extra2": {"type": "extra", "role": "role2"},
+                        "topic-extra3": {"type": "extra", "role": "role2"},
+                        "topic-extra": {"type": "extra", "role": "role1"},
+                        ".*": {"type": "input-pattern"},
+                        "example.*": {
+                            "type": "extra-pattern",
+                            "role": "another-pattern",
+                        },
+                    }
+                },
+            },
+        )
+        assert streams_app.get_input_topics() == ["example-input", "b", "a"]
+        assert streams_app.get_extra_input_topics() == {
+            "role1": ["topic-extra"],
+            "role2": ["topic-extra2", "topic-extra3"],
+        }
+
+    @pytest.mark.asyncio
+    async def test_get_output_topic(
+        self, config: PipelineConfig, handlers: ComponentHandlers
+    ):
+        streams_app = StreamsApp(
+            name=self.STREAMS_APP_NAME,
+            config=config,
+            handlers=handlers,
+            **{
+                "namespace": "test-namespace",
+                "app": {
+                    "streams": {"brokers": "fake-broker:9092"},
+                },
+                "from": {
+                    "topics": {
+                        "example-input": {"type": "input"},
+                    }
+                },
+                "to": {"topics": {"example-output": {"type": "output"}}},
+            },
+        )
+        assert streams_app.get_output_topic() == "example-output"

tests/pipeline/resources/pipeline-with-loop/defaults.yaml

@@ -0,0 +1,22 @@
+pipeline-component:
+  prefix: ""
+
+kubernetes-app:
+  namespace: example-namespace
+
+kafka-connector:
+  namespace: example-namespace
+
+kafka-app:
+  app:
+    streams:
+      brokers: 127.0.0.1:9092
+      schemaRegistryUrl: 127.0.0.1:8081
+      optimizeLeaveGroupBehavior: false
+
+streams-app:
+  app:
+    labels:
+      pipeline: ${pipeline_name}
+    streams:
+      optimizeLeaveGroupBehavior: false

tests/pipeline/resources/pipeline-with-loop/pipeline.yaml

@@ -0,0 +1,34 @@
+- type: producer
+  name: app1
+  app:
+    image: producer-image
+  to:
+    topics:
+      my-output-topic:
+        type: output
+
+- type: streams-app
+  name: app2
+  app:
+    image: app2-image
+  from:
+    topics:
+      my-output-topic:
+        type: input
+  to:
+    topics:
+      my-app2-topic:
+        type: output
+
+- type: streams-app
+  name: app3
+  app:
+    image: app3-image
+  from:
+    topics:
+      my-app2-topic:
+        type: input
+  to:
+    topics:
+      my-output-topic:
+        type: output