copy-paste from backup branch

bakdata · Jan 11, 2024 · e27b6fb · e27b6fb
1 parent 26e6893
commit e27b6fb
Show file tree

Hide file tree

Showing 68 changed files with 654 additions and 864 deletions.
diff --git a/docs/docs/resources/architecture/components-hierarchy.md b/docs/docs/resources/architecture/components-hierarchy.md
@@ -1,21 +1,17 @@
 ```mermaid
 flowchart BT
     KubernetesApp --> PipelineComponent
-    KafkaApp --> PipelineComponent
     HelmApp --> KubernetesApp
-    StreamsBootstrap --> HelmApp
+    KafkaApp --> HelmApp
     StreamsApp --> KafkaApp
-    StreamsApp --> StreamsBootstrap
     ProducerApp --> KafkaApp
-    ProducerApp --> StreamsBootstrap
     KafkaConnector --> PipelineComponent
     KafkaSourceConnector --> KafkaConnector
     KafkaSinkConnector --> KafkaConnector
 
     click KubernetesApp "/kpops/user/core-concepts/components/kubernetes-app"
     click HelmApp "/kpops/user/core-concepts/components/helm-app"
     click KafkaApp "/kpops/user/core-concepts/components/kafka-app"
-    click StreamsBootstrap "/kpops/user/core-concepts/components/streams-bootstrap"
     click StreamsApp "/kpops/user/core-concepts/components/streams-app"
     click ProducerApp "/kpops/user/core-concepts/components/producer-app"
     click KafkaConnector "/kpops/user/core-concepts/components/kafka-connector"

diff --git a/.../resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml b/.../resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml
@@ -2,10 +2,8 @@ helm-app.yaml:
 - app-helm-app.yaml
 - repo_config-helm-app.yaml
 kafka-app.yaml:
-- prefix.yaml
-- from_.yaml
-- to.yaml
 - app-kafka-app.yaml
+- version-kafka-app.yaml
 kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml

diff --git a/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml b/docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml
@@ -13,7 +13,10 @@ kpops_components_fields:
   - prefix
   - from_
   - to
+  - namespace
   - app
+  - repo_config
+  - version
   kafka-connector:
   - name
   - prefix
@@ -75,15 +78,6 @@ kpops_components_fields:
   - app
   - repo_config
   - version
-  streams-bootstrap:
-  - name
-  - prefix
-  - from_
-  - to
-  - namespace
-  - app
-  - repo_config
-  - version
 kpops_components_inheritance_ref:
   helm-app:
     bases:

diff --git a/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml b/docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml
@@ -10,7 +10,10 @@ kafka-app.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
+- namespace.yaml
 - app-kafka-app.yaml
+- repo_config-helm-app.yaml
+- version-kafka-app.yaml
 kafka-connector.yaml:
 - prefix.yaml
 - from_.yaml

diff --git a/docs/docs/resources/pipeline-components/helm-app.yaml b/docs/docs/resources/pipeline-components/helm-app.yaml
@@ -3,16 +3,16 @@
   name: helm-app # required
   # Pipeline prefix that will prefix every component name. If you wish to not
   # have any prefix you can specify an empty string.
-  prefix: ${pipeline_name}-
+  prefix: ${pipeline.name}-
   from: # Must not be null
     topics: # read from topic
-      ${pipeline_name}-input-topic:
+      ${pipeline.name}-input-topic:
         type: input # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra
-      ${pipeline_name}-input-pattern-topic:
+      ${pipeline.name}-input-pattern-topic:
         type: pattern # Implied to be an input pattern if `role` is undefined
-      ${pipeline_name}-extra-pattern-topic:
+      ${pipeline.name}-extra-pattern-topic:
         type: pattern # Implied to be an extra pattern if `role` is defined
         role: some-role
     components: # read from specific component
@@ -28,11 +28,11 @@
   # Topic(s) into which the component will write output
   to:
     topics:
-      ${pipeline_name}-output-topic:
+      ${pipeline.name}-output-topic:
         type: output # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
-      ${pipeline_name}-error-topic:
+      ${pipeline.name}-error-topic:
         type: error
         # Currently KPOps supports Avro and JSON schemas.
         key_schema: key-schema # must implement SchemaProvider to use

diff --git a/docs/docs/resources/pipeline-components/kafka-app.yaml b/docs/docs/resources/pipeline-components/kafka-app.yaml
@@ -4,16 +4,16 @@
   name: kafka-app # required
   # Pipeline prefix that will prefix every component name. If you wish to not
   # have any prefix you can specify an empty string.
-  prefix: ${pipeline_name}-
+  prefix: ${pipeline.name}-
   from: # Must not be null
     topics: # read from topic
-      ${pipeline_name}-input-topic:
+      ${pipeline.name}-input-topic:
         type: input # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra
-      ${pipeline_name}-input-pattern-topic:
+      ${pipeline.name}-input-pattern-topic:
         type: pattern # Implied to be an input pattern if `role` is undefined
-      ${pipeline_name}-extra-pattern-topic:
+      ${pipeline.name}-extra-pattern-topic:
         type: pattern # Implied to be an extra pattern if `role` is defined
         role: some-role
     components: # read from specific component
@@ -29,11 +29,11 @@
   # Topic(s) into which the component will write output
   to:
     topics:
-      ${pipeline_name}-output-topic:
+      ${pipeline.name}-output-topic:
         type: output # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
-      ${pipeline_name}-error-topic:
+      ${pipeline.name}-error-topic:
         type: error
         # Currently KPOps supports Avro and JSON schemas.
         key_schema: key-schema # must implement SchemaProvider to use
@@ -44,11 +44,23 @@
           cleanup.policy: compact
     models: # SchemaProvider is initiated with the values given here
       model: model
+  namespace: namespace # required
   # `app` can contain application-specific settings, hence  the user is free to
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${kafka_brokers} # required
-      schemaRegistryUrl: ${schema_registry_url}
+      brokers: ${config.kafka_brokers} # required
+      schemaRegistryUrl: ${config.schema_registry.url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
+  # Helm repository configuration (optional)
+  # If not set the helm repo add will not be called. Useful when using local Helm charts
+  repo_config:
+    repository_name: bakdata-streams-bootstrap # required
+    url: https://bakdata.github.io/streams-bootstrap/ # required
+    repo_auth_flags:
+      username: user
+      password: pass
+      ca_file: /home/user/path/to/ca-file
+      insecure_skip_tls_verify: false
+  version: "2.12.0" # Helm chart version
diff --git a/docs/docs/resources/pipeline-components/kafka-connector.yaml b/docs/docs/resources/pipeline-components/kafka-connector.yaml
@@ -2,16 +2,16 @@
   name: kafka-connector # required
   # Pipeline prefix that will prefix every component name. If you wish to not
   # have any prefix you can specify an empty string.
-  prefix: ${pipeline_name}-
+  prefix: ${pipeline.name}-
   from: # Must not be null
     topics: # read from topic
-      ${pipeline_name}-input-topic:
+      ${pipeline.name}-input-topic:
         type: input # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra
-      ${pipeline_name}-input-pattern-topic:
+      ${pipeline.name}-input-pattern-topic:
         type: pattern # Implied to be an input pattern if `role` is undefined
-      ${pipeline_name}-extra-pattern-topic:
+      ${pipeline.name}-extra-pattern-topic:
         type: pattern # Implied to be an extra pattern if `role` is defined
         role: some-role
     components: # read from specific component
@@ -27,11 +27,11 @@
   # Topic(s) into which the component will write output
   to:
     topics:
-      ${pipeline_name}-output-topic:
+      ${pipeline.name}-output-topic:
         type: output # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
-      ${pipeline_name}-error-topic:
+      ${pipeline.name}-error-topic:
         type: error
         # Currently KPOps supports Avro and JSON schemas.
         key_schema: key-schema # must implement SchemaProvider to use

diff --git a/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml b/docs/docs/resources/pipeline-components/kafka-sink-connector.yaml
@@ -3,16 +3,16 @@
   name: kafka-sink-connector # required
   # Pipeline prefix that will prefix every component name. If you wish to not
   # have any prefix you can specify an empty string.
-  prefix: ${pipeline_name}-
+  prefix: ${pipeline.name}-
   from: # Must not be null
     topics: # read from topic
-      ${pipeline_name}-input-topic:
+      ${pipeline.name}-input-topic:
         type: input # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra
-      ${pipeline_name}-input-pattern-topic:
+      ${pipeline.name}-input-pattern-topic:
         type: pattern # Implied to be an input pattern if `role` is undefined
-      ${pipeline_name}-extra-pattern-topic:
+      ${pipeline.name}-extra-pattern-topic:
         type: pattern # Implied to be an extra pattern if `role` is defined
         role: some-role
     components: # read from specific component
@@ -28,11 +28,11 @@
   # Topic(s) into which the component will write output
   to:
     topics:
-      ${pipeline_name}-output-topic:
+      ${pipeline.name}-output-topic:
         type: output # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
-      ${pipeline_name}-error-topic:
+      ${pipeline.name}-error-topic:
         type: error
         # Currently KPOps supports Avro and JSON schemas.
         key_schema: key-schema # must implement SchemaProvider to use

diff --git a/docs/docs/resources/pipeline-components/kafka-source-connector.yaml b/docs/docs/resources/pipeline-components/kafka-source-connector.yaml
@@ -3,17 +3,17 @@
   name: kafka-source-connector # required
   # Pipeline prefix that will prefix every component name. If you wish to not
   # have any prefix you can specify an empty string.
-  prefix: ${pipeline_name}-
+  prefix: ${pipeline.name}-
   # The source connector has no `from` section
   # from:
   # Topic(s) into which the component will write output
   to:
     topics:
-      ${pipeline_name}-output-topic:
+      ${pipeline.name}-output-topic:
         type: output # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
-      ${pipeline_name}-error-topic:
+      ${pipeline.name}-error-topic:
         type: error
         # Currently KPOps supports Avro and JSON schemas.
         key_schema: key-schema # must implement SchemaProvider to use

diff --git a/docs/docs/resources/pipeline-components/kubernetes-app.yaml b/docs/docs/resources/pipeline-components/kubernetes-app.yaml
@@ -3,16 +3,16 @@
   name: kubernetes-app # required
   # Pipeline prefix that will prefix every component name. If you wish to not
   # have any prefix you can specify an empty string.
-  prefix: ${pipeline_name}-
+  prefix: ${pipeline.name}-
   from: # Must not be null
     topics: # read from topic
-      ${pipeline_name}-input-topic:
+      ${pipeline.name}-input-topic:
         type: input # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra
-      ${pipeline_name}-input-pattern-topic:
+      ${pipeline.name}-input-pattern-topic:
         type: pattern # Implied to be an input pattern if `role` is undefined
-      ${pipeline_name}-extra-pattern-topic:
+      ${pipeline.name}-extra-pattern-topic:
         type: pattern # Implied to be an extra pattern if `role` is defined
         role: some-role
     components: # read from specific component
@@ -28,11 +28,11 @@
   # Topic(s) into which the component will write output
   to:
     topics:
-      ${pipeline_name}-output-topic:
+      ${pipeline.name}-output-topic:
         type: output # Implied when role is NOT specified
-      ${pipeline_name}-extra-topic:
+      ${pipeline.name}-extra-topic:
         role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
-      ${pipeline_name}-error-topic:
+      ${pipeline.name}-error-topic:
         type: error
         # Currently KPOps supports Avro and JSON schemas.
         key_schema: key-schema # must implement SchemaProvider to use