Refactor streams-bootstrap cleanup jobs as individual HelmApp

docs/docs/resources/architecture/components-hierarchy.md

@@ -1,17 +1,21 @@
 ```mermaid
 flowchart BT
     KubernetesApp --> PipelineComponent
+    KafkaApp --> PipelineComponent
     HelmApp --> KubernetesApp
-    KafkaApp --> HelmApp
+    StreamsBootstrap --> HelmApp
     StreamsApp --> KafkaApp
+    StreamsApp --> StreamsBootstrap
     ProducerApp --> KafkaApp
+    ProducerApp --> StreamsBootstrap
     KafkaConnector --> PipelineComponent
     KafkaSourceConnector --> KafkaConnector
     KafkaSinkConnector --> KafkaConnector
 
     click KubernetesApp "/kpops/user/core-concepts/components/kubernetes-app"
     click HelmApp "/kpops/user/core-concepts/components/helm-app"
     click KafkaApp "/kpops/user/core-concepts/components/kafka-app"
+    click StreamsBootstrap "/kpops/user/core-concepts/components/streams-bootstrap"
     click StreamsApp "/kpops/user/core-concepts/components/streams-app"
     click ProducerApp "/kpops/user/core-concepts/components/producer-app"
     click KafkaConnector "/kpops/user/core-concepts/components/kafka-connector"

docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml

@@ -2,8 +2,10 @@ helm-app.yaml:
 - app-helm-app.yaml
 - repo_config-helm-app.yaml
 kafka-app.yaml:
+- prefix.yaml
+- from_.yaml
+- to.yaml
 - app-kafka-app.yaml
-- version-kafka-app.yaml
 kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml

docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml

@@ -13,10 +13,7 @@ kpops_components_fields:
   - prefix
   - from_
   - to
-  - namespace
   - app
-  - repo_config
-  - version
   kafka-connector:
   - name
   - prefix
@@ -78,6 +75,15 @@ kpops_components_fields:
   - app
   - repo_config
   - version
+  streams-bootstrap:
+  - name
+  - prefix
+  - from_
+  - to
+  - namespace
+  - app
+  - repo_config
+  - version
 kpops_components_inheritance_ref:
   helm-app:
     bases:
@@ -87,10 +93,8 @@ kpops_components_inheritance_ref:
     - pipeline-component
   kafka-app:
     bases:
-    - helm-app
+    - pipeline-component
     parents:
-    - helm-app
-    - kubernetes-app
     - pipeline-component
   kafka-connector:
     bases:
@@ -120,16 +124,27 @@ kpops_components_inheritance_ref:
   producer-app:
     bases:
     - kafka-app
+    - streams-bootstrap
     parents:
     - kafka-app
+    - streams-bootstrap
     - helm-app
     - kubernetes-app
     - pipeline-component
   streams-app:
     bases:
     - kafka-app
+    - streams-bootstrap
     parents:
     - kafka-app
+    - streams-bootstrap
+    - helm-app
+    - kubernetes-app
+    - pipeline-component
+  streams-bootstrap:
+    bases:
+    - helm-app
+    parents:
     - helm-app
     - kubernetes-app
     - pipeline-component

docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml

@@ -10,10 +10,7 @@ kafka-app.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
-- namespace.yaml
 - app-kafka-app.yaml
-- repo_config-helm-app.yaml
-- version-kafka-app.yaml
 kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml

docs/docs/resources/pipeline-components/kafka-app.yaml

@@ -44,7 +44,6 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` can contain application-specific settings, hence  the user is free to
   # add the key-value pairs they need.
   app: # required
@@ -53,14 +52,3 @@
       schemaRegistryUrl: ${config.schema_registry.url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
-  # Helm repository configuration (optional)
-  # If not set the helm repo add will not be called. Useful when using local Helm charts
-  repo_config:
-    repository_name: bakdata-streams-bootstrap # required
-    url: https://bakdata.github.io/streams-bootstrap/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "2.12.0" # Helm chart version

docs/docs/resources/pipeline-components/pipeline.yaml

@@ -107,7 +107,6 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
-  namespace: namespace # required
   # `app` can contain application-specific settings, hence  the user is free to
   # add the key-value pairs they need.
   app: # required
@@ -116,17 +115,6 @@
       schemaRegistryUrl: ${config.schema_registry.url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
-  # Helm repository configuration (optional)
-  # If not set the helm repo add will not be called. Useful when using local Helm charts
-  repo_config:
-    repository_name: bakdata-streams-bootstrap # required
-    url: https://bakdata.github.io/streams-bootstrap/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "2.12.0" # Helm chart version
 # Kafka sink connector
 - type: kafka-sink-connector
   name: kafka-sink-connector # required

docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml

@@ -3,6 +3,48 @@
 # Parent of: ProducerApp, StreamsApp
 # Child of: KubernetesApp
 kafka-app:
+  # Pipeline prefix that will prefix every component name. If you wish to not
+  # have any prefix you can specify an empty string.
+  prefix: ${pipeline.name}-
+  from: # Must not be null
+    topics: # read from topic
+      ${pipeline.name}-input-topic:
+        type: input # Implied when role is NOT specified
+      ${pipeline.name}-extra-topic:
+        role: topic-role # Implies `type` to be extra
+      ${pipeline.name}-input-pattern-topic:
+        type: pattern # Implied to be an input pattern if `role` is undefined
+      ${pipeline.name}-extra-pattern-topic:
+        type: pattern # Implied to be an extra pattern if `role` is defined
+        role: some-role
+    components: # read from specific component
+      account-producer:
+        type: output # Implied when role is NOT specified
+      other-producer:
+        role: some-role # Implies `type` to be extra
+      component-as-input-pattern:
+        type: pattern # Implied to be an input pattern if `role` is undefined
+      component-as-extra-pattern:
+        type: pattern # Implied to be an extra pattern if `role` is defined
+        role: some-role
+  # Topic(s) into which the component will write output
+  to:
+    topics:
+      ${pipeline.name}-output-topic:
+        type: output # Implied when role is NOT specified
+      ${pipeline.name}-extra-topic:
+        role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
+      ${pipeline.name}-error-topic:
+        type: error
+        # Currently KPOps supports Avro and JSON schemas.
+        key_schema: key-schema # must implement SchemaProvider to use
+        value_schema: value-schema
+        partitions_count: 1
+        replication_factor: 1
+        configs: # https://kafka.apache.org/documentation/#topicconfigs
+          cleanup.policy: compact
+    models: # SchemaProvider is initiated with the values given here
+      model: model
   # `app` can contain application-specific settings, hence  the user is free to
   # add the key-value pairs they need.
   app: # required
@@ -11,4 +53,3 @@ kafka-app:
       schemaRegistryUrl: ${config.schema_registry.url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
-  version: "2.12.0" # Helm chart version

docs/docs/resources/pipeline-defaults/defaults.yaml

@@ -24,6 +24,48 @@ helm-app:
 # Parent of: ProducerApp, StreamsApp
 # Child of: KubernetesApp
 kafka-app:
+  # Pipeline prefix that will prefix every component name. If you wish to not
+  # have any prefix you can specify an empty string.
+  prefix: ${pipeline.name}-
+  from: # Must not be null
+    topics: # read from topic
+      ${pipeline.name}-input-topic:
+        type: input # Implied when role is NOT specified
+      ${pipeline.name}-extra-topic:
+        role: topic-role # Implies `type` to be extra
+      ${pipeline.name}-input-pattern-topic:
+        type: pattern # Implied to be an input pattern if `role` is undefined
+      ${pipeline.name}-extra-pattern-topic:
+        type: pattern # Implied to be an extra pattern if `role` is defined
+        role: some-role
+    components: # read from specific component
+      account-producer:
+        type: output # Implied when role is NOT specified
+      other-producer:
+        role: some-role # Implies `type` to be extra
+      component-as-input-pattern:
+        type: pattern # Implied to be an input pattern if `role` is undefined
+      component-as-extra-pattern:
+        type: pattern # Implied to be an extra pattern if `role` is defined
+        role: some-role
+  # Topic(s) into which the component will write output
+  to:
+    topics:
+      ${pipeline.name}-output-topic:
+        type: output # Implied when role is NOT specified
+      ${pipeline.name}-extra-topic:
+        role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
+      ${pipeline.name}-error-topic:
+        type: error
+        # Currently KPOps supports Avro and JSON schemas.
+        key_schema: key-schema # must implement SchemaProvider to use
+        value_schema: value-schema
+        partitions_count: 1
+        replication_factor: 1
+        configs: # https://kafka.apache.org/documentation/#topicconfigs
+          cleanup.policy: compact
+    models: # SchemaProvider is initiated with the values given here
+      model: model
   # `app` can contain application-specific settings, hence  the user is free to
   # add the key-value pairs they need.
   app: # required
@@ -32,7 +74,6 @@ kafka-app:
       schemaRegistryUrl: ${config.schema_registry.url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
-  version: "2.12.0" # Helm chart version
 # Kafka connector
 #
 # Parent of: KafkaSinkConnector, KafkaSourceConnector